History

Merge tag 'for-linus-iommufd' of git://git.kernel.org/pub/scm/linux/kernel/git/jgg/iommufd

Pull iommufd implementation from Jason Gunthorpe: "iommufd is the user API to control the IOMMU subsystem as it relates to managing IO page tables that point at user space memory. It takes over from drivers/vfio/vfio_iommu_type1.c (aka the VFIO container) which is the VFIO specific interface for a similar idea. We see a broad need for extended features, some being highly IOMMU device specific: - Binding iommu_domain's to PASID/SSID - Userspace IO page tables, for ARM, x86 and S390 - Kernel bypassed invalidation of user page tables - Re-use of the KVM page table in the IOMMU - Dirty page tracking in the IOMMU - Runtime Increase/Decrease of IOPTE size - PRI support with faults resolved in userspace Many of these HW features exist to support VM use cases - for instance the combination of PASID, PRI and Userspace IO Page Tables allows an implementation of DMA Shared Virtual Addressing (vSVA) within a guest. Dirty tracking enables VM live migration with SRIOV devices and PASID support allow creating "scalable IOV" devices, among other things. As these features are fundamental to a VM platform they need to be uniformly exposed to all the driver families that do DMA into VMs, which is currently VFIO and VDPA" For more background, see the extended explanations in Jason's pull request: https://lore.kernel.org/lkml/Y5dzTU8dlmXTbzoJ@nvidia.com/ * tag 'for-linus-iommufd' of git://git.kernel.org/pub/scm/linux/kernel/git/jgg/iommufd: (62 commits) iommufd: Change the order of MSI setup iommufd: Improve a few unclear bits of code iommufd: Fix comment typos vfio: Move vfio group specific code into group.c vfio: Refactor dma APIs for emulated devices vfio: Wrap vfio group module init/clean code into helpers vfio: Refactor vfio_device open and close vfio: Make vfio_device_open() truly device specific vfio: Swap order of vfio_device_container_register() and open_device() vfio: Set device->group in helper function vfio: Create wrappers for group register/unregister vfio: Move the sanity check of the group to vfio_create_group() vfio: Simplify vfio_create_group() iommufd: Allow iommufd to supply /dev/vfio/vfio vfio: Make vfio_container optionally compiled vfio: Move container related MODULE_ALIAS statements into container.c vfio-iommufd: Support iommufd for emulated VFIO devices vfio-iommufd: Support iommufd for physical VFIO devices vfio-iommufd: Allow iommufd to be used in place of a container fd vfio: Use IOMMU_CAP_ENFORCE_CACHE_COHERENCY for vfio_file_enforced_coherent() ...

Linus Torvalds committed 3 years ago

Commit 08cdc212 parents aa5ad10 d6c55c083 files · +12804 −1466Browse files

Unified · Split

83 files
  1. .clang-format +3 −0
  2. Documentation/userspace-api/index.rst +1 −0
  3. Documentation/userspace-api/ioctl/ioctl-number.rst +1 −0
  4. Documentation/userspace-api/iommufd.rst +223 −0
  5. MAINTAINERS +12 −0
  6. drivers/dma/idxd/cdev.c +1 −2
  7. drivers/dma/idxd/init.c +1 −24
  8. drivers/gpu/drm/i915/gvt/kvmgt.c +17 −4
  9. drivers/iommu/Kconfig +1 −0
  10. drivers/iommu/Makefile +2 −2
  11. drivers/iommu/amd/iommu.c +4 −10
  12. drivers/iommu/arm/arm-smmu-v3/arm-smmu-v3-sva.c +62 −42
  13. drivers/iommu/arm/arm-smmu-v3/arm-smmu-v3.c +18 −14
  14. drivers/iommu/arm/arm-smmu-v3/arm-smmu-v3.h +10 −13
  15. drivers/iommu/arm/arm-smmu/arm-smmu.c +0 −3
  16. drivers/iommu/arm/arm-smmu/qcom_iommu.c +1 −6
  17. drivers/iommu/fsl_pamu.c +1 −1
  18. drivers/iommu/fsl_pamu_domain.c +2 −2
  19. drivers/iommu/intel/dmar.c +7 −0
  20. drivers/iommu/intel/iommu.c +40 −16
  21. drivers/iommu/intel/iommu.h +12 −6
  22. drivers/iommu/intel/pasid.c +4 −2
  23. drivers/iommu/intel/svm.c +57 −88
  24. drivers/iommu/io-pgfault.c +17 −60
  25. drivers/iommu/iommu-sva-lib.c +0 −71
  26. drivers/iommu/iommu-sva.c +240 −0
  27. drivers/iommu/iommu-sva.h +11 −3
  28. drivers/iommu/iommu.c +297 −119
  29. drivers/iommu/iommufd/Kconfig +44 −0
  30. drivers/iommu/iommufd/Makefile +13 −0
  31. drivers/iommu/iommufd/device.c +772 −0
  32. drivers/iommu/iommufd/double_span.h +53 −0
  33. drivers/iommu/iommufd/hw_pagetable.c +57 −0
  34. drivers/iommu/iommufd/io_pagetable.c +1216 −0
  35. drivers/iommu/iommufd/io_pagetable.h +241 −0
  36. drivers/iommu/iommufd/ioas.c +398 −0
  37. drivers/iommu/iommufd/iommufd_private.h +307 −0
  38. drivers/iommu/iommufd/iommufd_test.h +93 −0
  39. drivers/iommu/iommufd/main.c +460 −0
  40. drivers/iommu/iommufd/pages.c +1977 −0
  41. drivers/iommu/iommufd/selftest.c +853 −0
  42. drivers/iommu/iommufd/vfio_compat.c +472 −0
  43. drivers/iommu/ipmmu-vmsa.c +0 −2
  44. drivers/iommu/mtk_iommu.c +2 −2
  45. drivers/iommu/omap-iommu.c +3 −3
  46. drivers/iommu/sprd-iommu.c +1 −3
  47. drivers/iommu/tegra-gart.c +1 −1
  48. drivers/iommu/virtio-iommu.c +3 −4
  49. drivers/misc/uacce/uacce.c +1 −1
  50. drivers/pci/ats.c +3 −0
  51. drivers/s390/cio/vfio_ccw_ops.c +3 −0
  52. drivers/s390/crypto/vfio_ap_ops.c +20 −1
  53. drivers/vfio/Kconfig +25 −11
  54. drivers/vfio/Makefile +4 −2
  55. drivers/vfio/container.c +42 −103
  56. drivers/vfio/fsl-mc/vfio_fsl_mc.c +3 −0
  57. drivers/vfio/group.c +877 −0
  58. drivers/vfio/iommufd.c +158 −0
  59. drivers/vfio/pci/hisilicon/hisi_acc_vfio_pci.c +6 −0
  60. drivers/vfio/pci/mlx5/main.c +3 −0
  61. drivers/vfio/pci/vfio_pci.c +3 −0
  62. drivers/vfio/platform/vfio_amba.c +3 −0
  63. drivers/vfio/platform/vfio_platform.c +3 −0
  64. drivers/vfio/vfio.h +114 −6
  65. drivers/vfio/vfio_main.c +193 −790
  66. include/linux/intel-svm.h +0 −13
  67. include/linux/interval_tree.h +58 −0
  68. include/linux/iommu.h +114 −31
  69. include/linux/iommufd.h +98 −0
  70. include/linux/sched/user.h +1 −1
  71. include/linux/vfio.h +39 −0
  72. include/uapi/linux/iommufd.h +347 −0
  73. kernel/user.c +1 −0
  74. lib/Kconfig +4 −0
  75. lib/interval_tree.c +132 −0
  76. scripts/kernel-doc +8 −4
  77. tools/testing/selftests/Makefile +1 −0
  78. tools/testing/selftests/iommu/.gitignore +3 −0
  79. tools/testing/selftests/iommu/Makefile +12 −0
  80. tools/testing/selftests/iommu/config +2 −0
  81. tools/testing/selftests/iommu/iommufd.c +1654 −0
  82. tools/testing/selftests/iommu/iommufd_fail_nth.c +580 −0
  83. tools/testing/selftests/iommu/iommufd_utils.h +278 −0
.clang-format+3−0
@@ -441,8 +441,11 @@ ForEachMacros: · 440 unmodified lines
441441 - 'inet_lhash2_for_each_icsk'
442442 - 'inet_lhash2_for_each_icsk_continue'
443443 - 'inet_lhash2_for_each_icsk_rcu'
444+ - 'interval_tree_for_each_double_span'
445+ - 'interval_tree_for_each_span'
444446 - 'intlist__for_each_entry'
445447 - 'intlist__for_each_entry_safe'
448+ - 'iopt_for_each_contig_area'
446449 - 'kcore_copy__for_each_phdr'
447450 - 'key_for_each'
448451 - 'key_for_each_safe'
Documentation/userspace-api/index.rst+1−0
@@ -25,6 +25,7 @@ place where this information is gathered. · 24 unmodified lines
2525 ebpf/index
2626 ioctl/index
2727 iommu
28+ iommufd
2829 media/index
2930 netlink/index
3031 sysfs-platform_profile
Documentation/userspace-api/ioctl/ioctl-number.rst+1−0
@@ -105,6 +105,7 @@ Code Seq# Include File Comments · 104 unmodified lines
105105 '8' all SNP8023 advanced NIC card
106106 <mailto:mcr@solidum.com>
107107 ';' 64-7F linux/vfio.h
108+';' 80-FF linux/iommufd.h
108109 '=' 00-3f uapi/linux/ptp_clock.h <mailto:richardcochran@gmail.com>
109110 '@' 00-0F linux/radeonfb.h conflict!
110111 '@' 00-0F drivers/video/aty/aty128fb.c conflict!
Documentation/userspace-api/iommufd.rst+223−0
@@ -0,0 +1,223 @@
1+.. SPDX-License-Identifier: GPL-2.0+
2+
3+=======
4+IOMMUFD
5+=======
6+
7+:Author: Jason Gunthorpe
8+:Author: Kevin Tian
9+
10+Overview
11+========
12+
13+IOMMUFD is the user API to control the IOMMU subsystem as it relates to managing
14+IO page tables from userspace using file descriptors. It intends to be general
15+and consumable by any driver that wants to expose DMA to userspace. These
16+drivers are eventually expected to deprecate any internal IOMMU logic
17+they may already/historically implement (e.g. vfio_iommu_type1.c).
18+
19+At minimum iommufd provides universal support of managing I/O address spaces and
20+I/O page tables for all IOMMUs, with room in the design to add non-generic
21+features to cater to specific hardware functionality.
22+
23+In this context the capital letter (IOMMUFD) refers to the subsystem while the
24+small letter (iommufd) refers to the file descriptors created via /dev/iommu for
25+use by userspace.
26+
27+Key Concepts
28+============
29+
30+User Visible Objects
31+--------------------
32+
33+Following IOMMUFD objects are exposed to userspace:
34+
35+- IOMMUFD_OBJ_IOAS, representing an I/O address space (IOAS), allowing map/unmap
36+ of user space memory into ranges of I/O Virtual Address (IOVA).
37+
38+ The IOAS is a functional replacement for the VFIO container, and like the VFIO
39+ container it copies an IOVA map to a list of iommu_domains held within it.
40+
41+- IOMMUFD_OBJ_DEVICE, representing a device that is bound to iommufd by an
42+ external driver.
43+
44+- IOMMUFD_OBJ_HW_PAGETABLE, representing an actual hardware I/O page table
45+ (i.e. a single struct iommu_domain) managed by the iommu driver.
46+
47+ The IOAS has a list of HW_PAGETABLES that share the same IOVA mapping and
48+ it will synchronize its mapping with each member HW_PAGETABLE.
49+
50+All user-visible objects are destroyed via the IOMMU_DESTROY uAPI.
51+
52+The diagram below shows relationship between user-visible objects and kernel
53+datastructures (external to iommufd), with numbers referred to operations
54+creating the objects and links::
55+
56+ _________________________________________________________
57+ | iommufd |
58+ | [1] |
59+ | _________________ |
60+ | | | |
61+ | | | |
62+ | | | |
63+ | | | |
64+ | | | |
65+ | | | |
66+ | | | [3] [2] |
67+ | | | ____________ __________ |
68+ | | IOAS |<--| |<------| | |
69+ | | | |HW_PAGETABLE| | DEVICE | |
70+ | | | |____________| |__________| |
71+ | | | | | |
72+ | | | | | |
73+ | | | | | |
74+ | | | | | |
75+ | | | | | |
76+ | |_________________| | | |
77+ | | | | |
78+ |_________|___________________|___________________|_______|
79+ | | |
80+ | _____v______ _______v_____
81+ | PFN storage | | | |
82+ |------------>|iommu_domain| |struct device|
83+ |____________| |_____________|
84+
85+1. IOMMUFD_OBJ_IOAS is created via the IOMMU_IOAS_ALLOC uAPI. An iommufd can
86+ hold multiple IOAS objects. IOAS is the most generic object and does not
87+ expose interfaces that are specific to single IOMMU drivers. All operations
88+ on the IOAS must operate equally on each of the iommu_domains inside of it.
89+
90+2. IOMMUFD_OBJ_DEVICE is created when an external driver calls the IOMMUFD kAPI
91+ to bind a device to an iommufd. The driver is expected to implement a set of
92+ ioctls to allow userspace to initiate the binding operation. Successful
93+ completion of this operation establishes the desired DMA ownership over the
94+ device. The driver must also set the driver_managed_dma flag and must not
95+ touch the device until this operation succeeds.
96+
97+3. IOMMUFD_OBJ_HW_PAGETABLE is created when an external driver calls the IOMMUFD
98+ kAPI to attach a bound device to an IOAS. Similarly the external driver uAPI
99+ allows userspace to initiate the attaching operation. If a compatible
100+ pagetable already exists then it is reused for the attachment. Otherwise a
101+ new pagetable object and iommu_domain is created. Successful completion of
102+ this operation sets up the linkages among IOAS, device and iommu_domain. Once
103+ this completes the device could do DMA.
104+
105+ Every iommu_domain inside the IOAS is also represented to userspace as a
106+ HW_PAGETABLE object.
107+
108+ .. note::
109+
110+ Future IOMMUFD updates will provide an API to create and manipulate the
111+ HW_PAGETABLE directly.
112+
113+A device can only bind to an iommufd due to DMA ownership claim and attach to at
114+most one IOAS object (no support of PASID yet).
115+
116+Kernel Datastructure
117+--------------------
118+
119+User visible objects are backed by following datastructures:
120+
121+- iommufd_ioas for IOMMUFD_OBJ_IOAS.
122+- iommufd_device for IOMMUFD_OBJ_DEVICE.
123+- iommufd_hw_pagetable for IOMMUFD_OBJ_HW_PAGETABLE.
124+
125+Several terminologies when looking at these datastructures:
126+
127+- Automatic domain - refers to an iommu domain created automatically when
128+ attaching a device to an IOAS object. This is compatible to the semantics of
129+ VFIO type1.
130+
131+- Manual domain - refers to an iommu domain designated by the user as the
132+ target pagetable to be attached to by a device. Though currently there are
133+ no uAPIs to directly create such domain, the datastructure and algorithms
134+ are ready for handling that use case.
135+
136+- In-kernel user - refers to something like a VFIO mdev that is using the
137+ IOMMUFD access interface to access the IOAS. This starts by creating an
138+ iommufd_access object that is similar to the domain binding a physical device
139+ would do. The access object will then allow converting IOVA ranges into struct
140+ page * lists, or doing direct read/write to an IOVA.
141+
142+iommufd_ioas serves as the metadata datastructure to manage how IOVA ranges are
143+mapped to memory pages, composed of:
144+
145+- struct io_pagetable holding the IOVA map
146+- struct iopt_area's representing populated portions of IOVA
147+- struct iopt_pages representing the storage of PFNs
148+- struct iommu_domain representing the IO page table in the IOMMU
149+- struct iopt_pages_access representing in-kernel users of PFNs
150+- struct xarray pinned_pfns holding a list of pages pinned by in-kernel users
151+
152+Each iopt_pages represents a logical linear array of full PFNs. The PFNs are
153+ultimately derived from userspace VAs via an mm_struct. Once they have been
154+pinned the PFNs are stored in IOPTEs of an iommu_domain or inside the pinned_pfns
155+xarray if they have been pinned through an iommufd_access.
156+
157+PFN have to be copied between all combinations of storage locations, depending
158+on what domains are present and what kinds of in-kernel "software access" users
159+exist. The mechanism ensures that a page is pinned only once.
160+
161+An io_pagetable is composed of iopt_areas pointing at iopt_pages, along with a
162+list of iommu_domains that mirror the IOVA to PFN map.
163+
164+Multiple io_pagetable-s, through their iopt_area-s, can share a single
165+iopt_pages which avoids multi-pinning and double accounting of page
166+consumption.
167+
168+iommufd_ioas is sharable between subsystems, e.g. VFIO and VDPA, as long as
169+devices managed by different subsystems are bound to a same iommufd.
170+
171+IOMMUFD User API
172+================
173+
174+.. kernel-doc:: include/uapi/linux/iommufd.h
175+
176+IOMMUFD Kernel API
177+==================
178+
179+The IOMMUFD kAPI is device-centric with group-related tricks managed behind the
180+scene. This allows the external drivers calling such kAPI to implement a simple
181+device-centric uAPI for connecting its device to an iommufd, instead of
182+explicitly imposing the group semantics in its uAPI as VFIO does.
183+
184+.. kernel-doc:: drivers/iommu/iommufd/device.c
185+ :export:
186+
187+.. kernel-doc:: drivers/iommu/iommufd/main.c
188+ :export:
189+
190+VFIO and IOMMUFD
191+----------------
192+
193+Connecting a VFIO device to iommufd can be done in two ways.
194+
195+First is a VFIO compatible way by directly implementing the /dev/vfio/vfio
196+container IOCTLs by mapping them into io_pagetable operations. Doing so allows
197+the use of iommufd in legacy VFIO applications by symlinking /dev/vfio/vfio to
198+/dev/iommufd or extending VFIO to SET_CONTAINER using an iommufd instead of a
199+container fd.
200+
201+The second approach directly extends VFIO to support a new set of device-centric
202+user API based on aforementioned IOMMUFD kernel API. It requires userspace
203+change but better matches the IOMMUFD API semantics and easier to support new
204+iommufd features when comparing it to the first approach.
205+
206+Currently both approaches are still work-in-progress.
207+
208+There are still a few gaps to be resolved to catch up with VFIO type1, as
209+documented in iommufd_vfio_check_extension().
210+
211+Future TODOs
212+============
213+
214+Currently IOMMUFD supports only kernel-managed I/O page table, similar to VFIO
215+type1. New features on the radar include:
216+
217+ - Binding iommu_domain's to PASID/SSID
218+ - Userspace page tables, for ARM, x86 and S390
219+ - Kernel bypass'd invalidation of user page tables
220+ - Re-use of the KVM page table in the IOMMU
221+ - Dirty page tracking in the IOMMU
222+ - Runtime Increase/Decrease of IOPTE size
223+ - PRI support with faults resolved in userspace
MAINTAINERS+12−0
@@ -10793,6 +10793,18 @@ F: drivers/iommu/dma-iommu.h · 10792 unmodified lines
1079310793 F: drivers/iommu/iova.c
1079410794 F: include/linux/iova.h
1079510795
10796+IOMMUFD
10797+M: Jason Gunthorpe <jgg@nvidia.com>
10798+M: Kevin Tian <kevin.tian@intel.com>
10799+L: iommu@lists.linux.dev
10800+S: Maintained
10801+T: git git://git.kernel.org/pub/scm/linux/kernel/git/jgg/iommufd.git
10802+F: Documentation/userspace-api/iommufd.rst
10803+F: drivers/iommu/iommufd/
10804+F: include/linux/iommufd.h
10805+F: include/uapi/linux/iommufd.h
10806+F: tools/testing/selftests/iommu/
10807+
1079610808 IOMMU SUBSYSTEM
1079710809 M: Joerg Roedel <joro@8bytes.org>
1079810810 M: Will Deacon <will@kernel.org>
drivers/dma/idxd/cdev.c+1−2
@@ -6,7 +6,6 @@ · 5 unmodified lines
66 #include <linux/pci.h>
77 #include <linux/device.h>
88 #include <linux/sched/task.h>
9−#include <linux/intel-svm.h>
109 #include <linux/io-64-nonatomic-lo-hi.h>
1110 #include <linux/cdev.h>
1211 #include <linux/fs.h>
@@ -100,7 +99,7 @@ static int idxd_cdev_open(struct inode *inode, struct file *filp) · 87 unmodified lines
10099 filp->private_data = ctx;
101100
102101 if (device_user_pasid_enabled(idxd)) {
103− sva = iommu_sva_bind_device(dev, current->mm, NULL);
102+ sva = iommu_sva_bind_device(dev, current->mm);
104103 if (IS_ERR(sva)) {
105104 rc = PTR_ERR(sva);
106105 dev_err(dev, "pasid allocation failed: %d\n", rc);
drivers/dma/idxd/init.c+1−24
@@ -14,7 +14,6 @@ · 13 unmodified lines
1414 #include <linux/io-64-nonatomic-lo-hi.h>
1515 #include <linux/device.h>
1616 #include <linux/idr.h>
17−#include <linux/intel-svm.h>
1817 #include <linux/iommu.h>
1918 #include <uapi/linux/idxd.h>
2019 #include <linux/dmaengine.h>
@@ -502,29 +501,7 @@ static struct idxd_device *idxd_alloc(struct pci_dev *pdev, struct idxd_driver_d · 481 unmodified lines
502501
503502 static int idxd_enable_system_pasid(struct idxd_device *idxd)
504503 {
505− int flags;
506− unsigned int pasid;
507− struct iommu_sva *sva;
508−
509− flags = SVM_FLAG_SUPERVISOR_MODE;
510−
511− sva = iommu_sva_bind_device(&idxd->pdev->dev, NULL, &flags);
512− if (IS_ERR(sva)) {
513− dev_warn(&idxd->pdev->dev,
514− "iommu sva bind failed: %ld\n", PTR_ERR(sva));
515− return PTR_ERR(sva);
516− }
517−
518− pasid = iommu_sva_get_pasid(sva);
519− if (pasid == IOMMU_PASID_INVALID) {
520− iommu_sva_unbind_device(sva);
521− return -ENODEV;
522− }
523−
524− idxd->sva = sva;
525− idxd->pasid = pasid;
526− dev_dbg(&idxd->pdev->dev, "system pasid: %u\n", pasid);
527− return 0;
504+ return -EOPNOTSUPP;
528505 }
529506
530507 static void idxd_disable_system_pasid(struct idxd_device *idxd)
drivers/gpu/drm/i915/gvt/kvmgt.c+17−4
@@ -669,9 +669,6 @@ static int intel_vgpu_open_device(struct vfio_device *vfio_dev) · 668 unmodified lines
669669
670670 vgpu->attached = true;
671671
672− kvmgt_protect_table_init(vgpu);
673− gvt_cache_init(vgpu);
674−
675672 vgpu->track_node.track_write = kvmgt_page_track_write;
676673 vgpu->track_node.track_flush_slot = kvmgt_page_track_flush_slot;
677674 kvm_get_kvm(vgpu->vfio_device.kvm);
@@ -715,6 +712,11 @@ static void intel_vgpu_close_device(struct vfio_device *vfio_dev) · 37 unmodified lines
715712 kvmgt_protect_table_destroy(vgpu);
716713 gvt_cache_destroy(vgpu);
717714
715+ WARN_ON(vgpu->nr_cache_entries);
716+
717+ vgpu->gfn_cache = RB_ROOT;
718+ vgpu->dma_addr_cache = RB_ROOT;
719+
718720 intel_vgpu_release_msi_eventfd_ctx(vgpu);
719721
720722 vgpu->attached = false;
@@ -1445,9 +1447,17 @@ static int intel_vgpu_init_dev(struct vfio_device *vfio_dev) · 724 unmodified lines
14451447 struct intel_vgpu *vgpu = vfio_dev_to_vgpu(vfio_dev);
14461448 struct intel_vgpu_type *type =
14471449 container_of(mdev->type, struct intel_vgpu_type, type);
1450+ int ret;
14481451
14491452 vgpu->gvt = kdev_to_i915(mdev->type->parent->dev)->gvt;
1450− return intel_gvt_create_vgpu(vgpu, type->conf);
1453+ ret = intel_gvt_create_vgpu(vgpu, type->conf);
1454+ if (ret)
1455+ return ret;
1456+
1457+ kvmgt_protect_table_init(vgpu);
1458+ gvt_cache_init(vgpu);
1459+
1460+ return 0;
14511461 }
14521462
14531463 static void intel_vgpu_release_dev(struct vfio_device *vfio_dev)
@@ -1468,6 +1478,9 @@ static const struct vfio_device_ops intel_vgpu_dev_ops = { · 14 unmodified lines
14681478 .mmap = intel_vgpu_mmap,
14691479 .ioctl = intel_vgpu_ioctl,
14701480 .dma_unmap = intel_vgpu_dma_unmap,
1481+ .bind_iommufd = vfio_iommufd_emulated_bind,
1482+ .unbind_iommufd = vfio_iommufd_emulated_unbind,
1483+ .attach_ioas = vfio_iommufd_emulated_attach_ioas,
14711484 };
14721485
14731486 static int intel_vgpu_probe(struct mdev_device *mdev)
drivers/iommu/Kconfig+1−0
@@ -188,6 +188,7 @@ config MSM_IOMMU · 187 unmodified lines
188188
189189 source "drivers/iommu/amd/Kconfig"
190190 source "drivers/iommu/intel/Kconfig"
191+source "drivers/iommu/iommufd/Kconfig"
191192
192193 config IRQ_REMAP
193194 bool "Support for Interrupt Remapping"
drivers/iommu/Makefile+2−2
@@ -1,5 +1,5 @@
11 # SPDX-License-Identifier: GPL-2.0
2−obj-y += amd/ intel/ arm/
2+obj-y += amd/ intel/ arm/ iommufd/
33 obj-$(CONFIG_IOMMU_API) += iommu.o
44 obj-$(CONFIG_IOMMU_API) += iommu-traces.o
55 obj-$(CONFIG_IOMMU_API) += iommu-sysfs.o
@@ -28,6 +28,6 @@ obj-$(CONFIG_FSL_PAMU) += fsl_pamu.o fsl_pamu_domain.o · 22 unmodified lines
2828 obj-$(CONFIG_S390_IOMMU) += s390-iommu.o
2929 obj-$(CONFIG_HYPERV_IOMMU) += hyperv-iommu.o
3030 obj-$(CONFIG_VIRTIO_IOMMU) += virtio-iommu.o
31−obj-$(CONFIG_IOMMU_SVA) += iommu-sva-lib.o io-pgfault.o
31+obj-$(CONFIG_IOMMU_SVA) += iommu-sva.o io-pgfault.o
3232 obj-$(CONFIG_SPRD_IOMMU) += sprd-iommu.o
3333 obj-$(CONFIG_APPLE_DART) += apple-dart.o
drivers/iommu/amd/iommu.c+4−10
@@ -2155,21 +2155,13 @@ static void amd_iommu_detach_device(struct iommu_domain *dom, · 2154 unmodified lines
21552155 static int amd_iommu_attach_device(struct iommu_domain *dom,
21562156 struct device *dev)
21572157 {
2158+ struct iommu_dev_data *dev_data = dev_iommu_priv_get(dev);
21582159 struct protection_domain *domain = to_pdomain(dom);
2159− struct iommu_dev_data *dev_data;
2160− struct amd_iommu *iommu;
2160+ struct amd_iommu *iommu = rlookup_amd_iommu(dev);
21612161 int ret;
21622162
2163− if (!check_device(dev))
2164− return -EINVAL;
2165−
2166− dev_data = dev_iommu_priv_get(dev);
21672163 dev_data->defer_attach = false;
21682164
2169− iommu = rlookup_amd_iommu(dev);
2170− if (!iommu)
2171− return -EINVAL;
2172−
21732165 if (dev_data->domain)
21742166 detach_device(dev);
21752167
@@ -2286,6 +2278,8 @@ static bool amd_iommu_capable(struct device *dev, enum iommu_cap cap) · 110 unmodified lines
22862278 return false;
22872279 case IOMMU_CAP_PRE_BOOT_PROTECTION:
22882280 return amdr_ivrs_remap_support;
2281+ case IOMMU_CAP_ENFORCE_CACHE_COHERENCY:
2282+ return true;
22892283 default:
22902284 break;
22912285 }
drivers/iommu/arm/arm-smmu-v3/arm-smmu-v3-sva.c+62−42
@@ -10,7 +10,7 @@ · 9 unmodified lines
1010 #include <linux/slab.h>
1111
1212 #include "arm-smmu-v3.h"
13−#include "../../iommu-sva-lib.h"
13+#include "../../iommu-sva.h"
1414 #include "../../io-pgtable-arm.h"
1515
1616 struct arm_smmu_mmu_notifier {
@@ -344,11 +344,6 @@ __arm_smmu_sva_bind(struct device *dev, struct mm_struct *mm) · 327 unmodified lines
344344 if (!bond)
345345 return ERR_PTR(-ENOMEM);
346346
347− /* Allocate a PASID for this mm if necessary */
348− ret = iommu_sva_alloc_pasid(mm, 1, (1U << master->ssid_bits) - 1);
349− if (ret)
350− goto err_free_bond;
351−
352347 bond->mm = mm;
353348 bond->sva.dev = dev;
354349 refcount_set(&bond->refs, 1);
@@ -367,42 +362,6 @@ err_free_bond: · 12 unmodified lines
367362 return ERR_PTR(ret);
368363 }
369364
370−struct iommu_sva *
371−arm_smmu_sva_bind(struct device *dev, struct mm_struct *mm, void *drvdata)
372−{
373− struct iommu_sva *handle;
374− struct iommu_domain *domain = iommu_get_domain_for_dev(dev);
375− struct arm_smmu_domain *smmu_domain = to_smmu_domain(domain);
376−
377− if (smmu_domain->stage != ARM_SMMU_DOMAIN_S1)
378− return ERR_PTR(-EINVAL);
379−
380− mutex_lock(&sva_lock);
381− handle = __arm_smmu_sva_bind(dev, mm);
382− mutex_unlock(&sva_lock);
383− return handle;
384−}
385−
386−void arm_smmu_sva_unbind(struct iommu_sva *handle)
387−{
388− struct arm_smmu_bond *bond = sva_to_bond(handle);
389−
390− mutex_lock(&sva_lock);
391− if (refcount_dec_and_test(&bond->refs)) {
392− list_del(&bond->list);
393− arm_smmu_mmu_notifier_put(bond->smmu_mn);
394− kfree(bond);
395− }
396− mutex_unlock(&sva_lock);
397−}
398−
399−u32 arm_smmu_sva_get_pasid(struct iommu_sva *handle)
400−{
401− struct arm_smmu_bond *bond = sva_to_bond(handle);
402−
403− return bond->mm->pasid;
404−}
405−
406365 bool arm_smmu_sva_supported(struct arm_smmu_device *smmu)
407366 {
408367 unsigned long reg, fld;
@@ -550,3 +509,64 @@ void arm_smmu_sva_notifier_synchronize(void) · 141 unmodified lines
550509 */
551510 mmu_notifier_synchronize();
552511 }
512+
513+void arm_smmu_sva_remove_dev_pasid(struct iommu_domain *domain,
514+ struct device *dev, ioasid_t id)
515+{
516+ struct mm_struct *mm = domain->mm;
517+ struct arm_smmu_bond *bond = NULL, *t;
518+ struct arm_smmu_master *master = dev_iommu_priv_get(dev);
519+
520+ mutex_lock(&sva_lock);
521+ list_for_each_entry(t, &master->bonds, list) {
522+ if (t->mm == mm) {
523+ bond = t;
524+ break;
525+ }
526+ }
527+
528+ if (!WARN_ON(!bond) && refcount_dec_and_test(&bond->refs)) {
529+ list_del(&bond->list);
530+ arm_smmu_mmu_notifier_put(bond->smmu_mn);
531+ kfree(bond);
532+ }
533+ mutex_unlock(&sva_lock);
534+}
535+
536+static int arm_smmu_sva_set_dev_pasid(struct iommu_domain *domain,
537+ struct device *dev, ioasid_t id)
538+{
539+ int ret = 0;
540+ struct iommu_sva *handle;
541+ struct mm_struct *mm = domain->mm;
542+
543+ mutex_lock(&sva_lock);
544+ handle = __arm_smmu_sva_bind(dev, mm);
545+ if (IS_ERR(handle))
546+ ret = PTR_ERR(handle);
547+ mutex_unlock(&sva_lock);
548+
549+ return ret;
550+}
551+
552+static void arm_smmu_sva_domain_free(struct iommu_domain *domain)
553+{
554+ kfree(domain);
555+}
556+
557+static const struct iommu_domain_ops arm_smmu_sva_domain_ops = {
558+ .set_dev_pasid = arm_smmu_sva_set_dev_pasid,
559+ .free = arm_smmu_sva_domain_free
560+};
561+
562+struct iommu_domain *arm_smmu_sva_domain_alloc(void)
563+{
564+ struct iommu_domain *domain;
565+
566+ domain = kzalloc(sizeof(*domain), GFP_KERNEL);
567+ if (!domain)
568+ return NULL;
569+ domain->ops = &arm_smmu_sva_domain_ops;
570+
571+ return domain;
572+}
drivers/iommu/arm/arm-smmu-v3/arm-smmu-v3.c+18−14
@@ -29,7 +29,7 @@ · 28 unmodified lines
2929
3030 #include "arm-smmu-v3.h"
3131 #include "../../dma-iommu.h"
32−#include "../../iommu-sva-lib.h"
32+#include "../../iommu-sva.h"
3333
3434 static bool disable_bypass = true;
3535 module_param(disable_bypass, bool, 0444);
@@ -2009,6 +2009,9 @@ static struct iommu_domain *arm_smmu_domain_alloc(unsigned type) · 1973 unmodified lines
20092009 {
20102010 struct arm_smmu_domain *smmu_domain;
20112011
2012+ if (type == IOMMU_DOMAIN_SVA)
2013+ return arm_smmu_sva_domain_alloc();
2014+
20122015 if (type != IOMMU_DOMAIN_UNMANAGED &&
20132016 type != IOMMU_DOMAIN_DMA &&
20142017 type != IOMMU_DOMAIN_DMA_FQ &&
@@ -2430,23 +2433,14 @@ static int arm_smmu_attach_dev(struct iommu_domain *domain, struct device *dev) · 415 unmodified lines
24302433 goto out_unlock;
24312434 }
24322435 } else if (smmu_domain->smmu != smmu) {
2433− dev_err(dev,
2434− "cannot attach to SMMU %s (upstream of %s)\n",
2435− dev_name(smmu_domain->smmu->dev),
2436− dev_name(smmu->dev));
2437− ret = -ENXIO;
2436+ ret = -EINVAL;
24382437 goto out_unlock;
24392438 } else if (smmu_domain->stage == ARM_SMMU_DOMAIN_S1 &&
24402439 master->ssid_bits != smmu_domain->s1_cfg.s1cdmax) {
2441− dev_err(dev,
2442− "cannot attach to incompatible domain (%u SSID bits != %u)\n",
2443− smmu_domain->s1_cfg.s1cdmax, master->ssid_bits);
24442440 ret = -EINVAL;
24452441 goto out_unlock;
24462442 } else if (smmu_domain->stage == ARM_SMMU_DOMAIN_S1 &&
24472443 smmu_domain->stall_enabled != master->stall_enabled) {
2448− dev_err(dev, "cannot attach to stall-%s domain\n",
2449− smmu_domain->stall_enabled ? "enabled" : "disabled");
24502444 ret = -EINVAL;
24512445 goto out_unlock;
24522446 }
@@ -2838,6 +2832,17 @@ static int arm_smmu_def_domain_type(struct device *dev) · 385 unmodified lines
28382832 return 0;
28392833 }
28402834
2835+static void arm_smmu_remove_dev_pasid(struct device *dev, ioasid_t pasid)
2836+{
2837+ struct iommu_domain *domain;
2838+
2839+ domain = iommu_get_domain_for_dev_pasid(dev, pasid, IOMMU_DOMAIN_SVA);
2840+ if (WARN_ON(IS_ERR(domain)) || !domain)
2841+ return;
2842+
2843+ arm_smmu_sva_remove_dev_pasid(domain, dev, pasid);
2844+}
2845+
28412846 static struct iommu_ops arm_smmu_ops = {
28422847 .capable = arm_smmu_capable,
28432848 .domain_alloc = arm_smmu_domain_alloc,
@@ -2846,11 +2851,9 @@ static struct iommu_ops arm_smmu_ops = { · 2 unmodified lines
28462851 .device_group = arm_smmu_device_group,
28472852 .of_xlate = arm_smmu_of_xlate,
28482853 .get_resv_regions = arm_smmu_get_resv_regions,
2854+ .remove_dev_pasid = arm_smmu_remove_dev_pasid,
28492855 .dev_enable_feat = arm_smmu_dev_enable_feature,
28502856 .dev_disable_feat = arm_smmu_dev_disable_feature,
2851− .sva_bind = arm_smmu_sva_bind,
2852− .sva_unbind = arm_smmu_sva_unbind,
2853− .sva_get_pasid = arm_smmu_sva_get_pasid,
28542857 .page_response = arm_smmu_page_response,
28552858 .def_domain_type = arm_smmu_def_domain_type,
28562859 .pgsize_bitmap = -1UL, /* Restricted during device attach */
@@ -3543,6 +3546,7 @@ static int arm_smmu_device_hw_probe(struct arm_smmu_device *smmu) · 686 unmodified lines
35433546 /* SID/SSID sizes */
35443547 smmu->ssid_bits = FIELD_GET(IDR1_SSIDSIZE, reg);
35453548 smmu->sid_bits = FIELD_GET(IDR1_SIDSIZE, reg);
3549+ smmu->iommu.max_pasids = 1UL << smmu->ssid_bits;
35463550
35473551 /*
35483552 * If the SMMU supports fewer bits than would fill a single L2 stream
drivers/iommu/arm/arm-smmu-v3/arm-smmu-v3.h+10−13
@@ -754,11 +754,10 @@ bool arm_smmu_master_sva_enabled(struct arm_smmu_master *master); · 753 unmodified lines
754754 int arm_smmu_master_enable_sva(struct arm_smmu_master *master);
755755 int arm_smmu_master_disable_sva(struct arm_smmu_master *master);
756756 bool arm_smmu_master_iopf_supported(struct arm_smmu_master *master);
757−struct iommu_sva *arm_smmu_sva_bind(struct device *dev, struct mm_struct *mm,
758− void *drvdata);
759−void arm_smmu_sva_unbind(struct iommu_sva *handle);
760−u32 arm_smmu_sva_get_pasid(struct iommu_sva *handle);
761757 void arm_smmu_sva_notifier_synchronize(void);
758+struct iommu_domain *arm_smmu_sva_domain_alloc(void);
759+void arm_smmu_sva_remove_dev_pasid(struct iommu_domain *domain,
760+ struct device *dev, ioasid_t id);
762761 #else /* CONFIG_ARM_SMMU_V3_SVA */
763762 static inline bool arm_smmu_sva_supported(struct arm_smmu_device *smmu)
764763 {
@@ -790,19 +789,17 @@ static inline bool arm_smmu_master_iopf_supported(struct arm_smmu_master *master · 25 unmodified lines
790789 return false;
791790 }
792791
793−static inline struct iommu_sva *
794−arm_smmu_sva_bind(struct device *dev, struct mm_struct *mm, void *drvdata)
792+static inline void arm_smmu_sva_notifier_synchronize(void) {}
793+
794+static inline struct iommu_domain *arm_smmu_sva_domain_alloc(void)
795795 {
796− return ERR_PTR(-ENODEV);
796+ return NULL;
797797 }
798798
799−static inline void arm_smmu_sva_unbind(struct iommu_sva *handle) {}
800−
801−static inline u32 arm_smmu_sva_get_pasid(struct iommu_sva *handle)
799+static inline void arm_smmu_sva_remove_dev_pasid(struct iommu_domain *domain,
800+ struct device *dev,
801+ ioasid_t id)
802802 {
803− return IOMMU_PASID_INVALID;
804803 }
805−
806−static inline void arm_smmu_sva_notifier_synchronize(void) {}
807804 #endif /* CONFIG_ARM_SMMU_V3_SVA */
808805 #endif /* _ARM_SMMU_V3_H */
drivers/iommu/arm/arm-smmu/arm-smmu.c+0−3
@@ -1150,9 +1150,6 @@ static int arm_smmu_attach_dev(struct iommu_domain *domain, struct device *dev) · 1149 unmodified lines
11501150 * different SMMUs.
11511151 */
11521152 if (smmu_domain->smmu != smmu) {
1153− dev_err(dev,
1154− "cannot attach to SMMU %s whilst already attached to domain on SMMU %s\n",
1155− dev_name(smmu_domain->smmu->dev), dev_name(smmu->dev));
11561153 ret = -EINVAL;
11571154 goto rpm_put;
11581155 }
drivers/iommu/arm/arm-smmu/qcom_iommu.c+1−6
@@ -381,13 +381,8 @@ static int qcom_iommu_attach_dev(struct iommu_domain *domain, struct device *dev · 380 unmodified lines
381381 * Sanity check the domain. We don't support domains across
382382 * different IOMMUs.
383383 */
384− if (qcom_domain->iommu != qcom_iommu) {
385− dev_err(dev, "cannot attach to IOMMU %s while already "
386− "attached to domain on IOMMU %s\n",
387− dev_name(qcom_domain->iommu->dev),
388− dev_name(qcom_iommu->dev));
384+ if (qcom_domain->iommu != qcom_iommu)
389385 return -EINVAL;
390− }
391386
392387 return 0;
393388 }
drivers/iommu/fsl_pamu.c+1−1
@@ -211,7 +211,7 @@ int pamu_config_ppaace(int liodn, u32 omi, u32 stashid, int prot) · 210 unmodified lines
211211 ppaace->op_encode.index_ot.omi = omi;
212212 } else if (~omi != 0) {
213213 pr_debug("bad operation mapping index: %d\n", omi);
214− return -EINVAL;
214+ return -ENODEV;
215215 }
216216
217217 /* configure stash id */
drivers/iommu/fsl_pamu_domain.c+2−2
@@ -258,7 +258,7 @@ static int fsl_pamu_attach_device(struct iommu_domain *domain, · 257 unmodified lines
258258 liodn = of_get_property(dev->of_node, "fsl,liodn", &len);
259259 if (!liodn) {
260260 pr_debug("missing fsl,liodn property at %pOF\n", dev->of_node);
261− return -EINVAL;
261+ return -ENODEV;
262262 }
263263
264264 spin_lock_irqsave(&dma_domain->domain_lock, flags);
@@ -267,7 +267,7 @@ static int fsl_pamu_attach_device(struct iommu_domain *domain, · 2 unmodified lines
267267 if (liodn[i] >= PAACE_NUMBER_ENTRIES) {
268268 pr_debug("Invalid liodn %d, attach device failed for %pOF\n",
269269 liodn[i], dev->of_node);
270− ret = -EINVAL;
270+ ret = -ENODEV;
271271 break;
272272 }
273273
drivers/iommu/intel/dmar.c+7−0
@@ -1105,6 +1105,13 @@ static int alloc_iommu(struct dmar_drhd_unit *drhd) · 1104 unmodified lines
11051105
11061106 raw_spin_lock_init(&iommu->register_lock);
11071107
1108+ /*
1109+ * A value of N in PSS field of eCap register indicates hardware
1110+ * supports PASID field of N+1 bits.
1111+ */
1112+ if (pasid_supported(iommu))
1113+ iommu->iommu.max_pasids = 2UL << ecap_pss(iommu->ecap);
1114+
11081115 /*
11091116 * This is only for hotplug; at boot time intel_iommu_enabled won't
11101117 * be set yet. When intel_iommu_init() runs, it registers the units
drivers/iommu/intel/iommu.c+40−16
@@ -27,7 +27,7 @@ · 26 unmodified lines
2727 #include "iommu.h"
2828 #include "../dma-iommu.h"
2929 #include "../irq_remapping.h"
30−#include "../iommu-sva-lib.h"
30+#include "../iommu-sva.h"
3131 #include "pasid.h"
3232 #include "cap_audit.h"
3333
@@ -4188,6 +4188,8 @@ static struct iommu_domain *intel_iommu_domain_alloc(unsigned type) · 4154 unmodified lines
41884188 return domain;
41894189 case IOMMU_DOMAIN_IDENTITY:
41904190 return &si_domain->domain;
4191+ case IOMMU_DOMAIN_SVA:
4192+ return intel_svm_domain_alloc();
41914193 default:
41924194 return NULL;
41934195 }
@@ -4213,19 +4215,15 @@ static int prepare_domain_attach_device(struct iommu_domain *domain, · 19 unmodified lines
42134215 return -ENODEV;
42144216
42154217 if (dmar_domain->force_snooping && !ecap_sc_support(iommu->ecap))
4216− return -EOPNOTSUPP;
4218+ return -EINVAL;
42174219
42184220 /* check if this iommu agaw is sufficient for max mapped address */
42194221 addr_width = agaw_to_width(iommu->agaw);
42204222 if (addr_width > cap_mgaw(iommu->cap))
42214223 addr_width = cap_mgaw(iommu->cap);
42224224
4223− if (dmar_domain->max_addr > (1LL << addr_width)) {
4224− dev_err(dev, "%s: iommu width (%d) is not "
4225− "sufficient for the mapped address (%llx)\n",
4226− __func__, addr_width, dmar_domain->max_addr);
4227− return -EFAULT;
4228− }
4225+ if (dmar_domain->max_addr > (1LL << addr_width))
4226+ return -EINVAL;
42294227 dmar_domain->gaw = addr_width;
42304228
42314229 /*
@@ -4471,14 +4469,20 @@ static bool intel_iommu_enforce_cache_coherency(struct iommu_domain *domain) · 239 unmodified lines
44714469
44724470 static bool intel_iommu_capable(struct device *dev, enum iommu_cap cap)
44734471 {
4474− if (cap == IOMMU_CAP_CACHE_COHERENCY)
4472+ struct device_domain_info *info = dev_iommu_priv_get(dev);
4473+
4474+ switch (cap) {
4475+ case IOMMU_CAP_CACHE_COHERENCY:
44754476 return true;
4476− if (cap == IOMMU_CAP_INTR_REMAP)
4477+ case IOMMU_CAP_INTR_REMAP:
44774478 return irq_remapping_enabled == 1;
4478− if (cap == IOMMU_CAP_PRE_BOOT_PROTECTION)
4479+ case IOMMU_CAP_PRE_BOOT_PROTECTION:
44794480 return dmar_platform_optin();
4480−
4481− return false;
4481+ case IOMMU_CAP_ENFORCE_CACHE_COHERENCY:
4482+ return ecap_sc_support(info->iommu->ecap);
4483+ default:
4484+ return false;
4485+ }
44824486 }
44834487
44844488 static struct iommu_device *intel_iommu_probe_device(struct device *dev)
@@ -4732,6 +4736,28 @@ static void intel_iommu_iotlb_sync_map(struct iommu_domain *domain, · 247 unmodified lines
47324736 __mapping_notify_one(info->iommu, dmar_domain, pfn, pages);
47334737 }
47344738
4739+static void intel_iommu_remove_dev_pasid(struct device *dev, ioasid_t pasid)
4740+{
4741+ struct intel_iommu *iommu = device_to_iommu(dev, NULL, NULL);
4742+ struct iommu_domain *domain;
4743+
4744+ /* Domain type specific cleanup: */
4745+ domain = iommu_get_domain_for_dev_pasid(dev, pasid, 0);
4746+ if (domain) {
4747+ switch (domain->type) {
4748+ case IOMMU_DOMAIN_SVA:
4749+ intel_svm_remove_dev_pasid(dev, pasid);
4750+ break;
4751+ default:
4752+ /* should never reach here */
4753+ WARN_ON(1);
4754+ break;
4755+ }
4756+ }
4757+
4758+ intel_pasid_tear_down_entry(iommu, dev, pasid, false);
4759+}
4760+
47354761 const struct iommu_ops intel_iommu_ops = {
47364762 .capable = intel_iommu_capable,
47374763 .domain_alloc = intel_iommu_domain_alloc,
@@ -4744,11 +4770,9 @@ const struct iommu_ops intel_iommu_ops = { · 6 unmodified lines
47444770 .dev_disable_feat = intel_iommu_dev_disable_feat,
47454771 .is_attach_deferred = intel_iommu_is_attach_deferred,
47464772 .def_domain_type = device_def_domain_type,
4773+ .remove_dev_pasid = intel_iommu_remove_dev_pasid,
47474774 .pgsize_bitmap = SZ_4K,
47484775 #ifdef CONFIG_INTEL_IOMMU_SVM
4749− .sva_bind = intel_svm_bind,
4750− .sva_unbind = intel_svm_unbind,
4751− .sva_get_pasid = intel_svm_get_pasid,
47524776 .page_response = intel_svm_page_response,
47534777 #endif
47544778 .default_domain_ops = &(const struct iommu_domain_ops) {
drivers/iommu/intel/iommu.h+12−6
@@ -480,8 +480,6 @@ enum { · 479 unmodified lines
480480 #define VTD_FLAG_IRQ_REMAP_PRE_ENABLED (1 << 1)
481481 #define VTD_FLAG_SVM_CAPABLE (1 << 2)
482482
483−extern int intel_iommu_sm;
484−
485483 #define sm_supported(iommu) (intel_iommu_sm && ecap_smts((iommu)->ecap))
486484 #define pasid_supported(iommu) (sm_supported(iommu) && \
487485 ecap_pasid((iommu)->ecap))
@@ -753,12 +751,10 @@ struct intel_iommu *device_to_iommu(struct device *dev, u8 *bus, u8 *devfn); · 265 unmodified lines
753751 extern void intel_svm_check(struct intel_iommu *iommu);
754752 extern int intel_svm_enable_prq(struct intel_iommu *iommu);
755753 extern int intel_svm_finish_prq(struct intel_iommu *iommu);
756−struct iommu_sva *intel_svm_bind(struct device *dev, struct mm_struct *mm,
757− void *drvdata);
758−void intel_svm_unbind(struct iommu_sva *handle);
759−u32 intel_svm_get_pasid(struct iommu_sva *handle);
760754 int intel_svm_page_response(struct device *dev, struct iommu_fault_event *evt,
761755 struct iommu_page_response *msg);
756+struct iommu_domain *intel_svm_domain_alloc(void);
757+void intel_svm_remove_dev_pasid(struct device *dev, ioasid_t pasid);
762758
763759 struct intel_svm_dev {
764760 struct list_head list;
@@ -783,6 +779,14 @@ struct intel_svm { · 18 unmodified lines
783779 };
784780 #else
785781 static inline void intel_svm_check(struct intel_iommu *iommu) {}
782+static inline struct iommu_domain *intel_svm_domain_alloc(void)
783+{
784+ return NULL;
785+}
786+
787+static inline void intel_svm_remove_dev_pasid(struct device *dev, ioasid_t pasid)
788+{
789+}
786790 #endif
787791
788792 #ifdef CONFIG_INTEL_IOMMU_DEBUGFS
@@ -798,6 +802,7 @@ struct context_entry *iommu_context_addr(struct intel_iommu *iommu, u8 bus, · 9 unmodified lines
798802 extern const struct iommu_ops intel_iommu_ops;
799803
800804 #ifdef CONFIG_INTEL_IOMMU
805+extern int intel_iommu_sm;
801806 extern int iommu_calculate_agaw(struct intel_iommu *iommu);
802807 extern int iommu_calculate_max_sagaw(struct intel_iommu *iommu);
803808 extern int dmar_disabled;
@@ -813,6 +818,7 @@ static inline int iommu_calculate_max_sagaw(struct intel_iommu *iommu) · 9 unmodified lines
813818 }
814819 #define dmar_disabled (1)
815820 #define intel_iommu_enabled (0)
821+#define intel_iommu_sm (0)
816822 #endif
817823
818824 static inline const char *decode_prq_descriptor(char *str, size_t size,
drivers/iommu/intel/pasid.c+4−2
@@ -101,8 +101,10 @@ int intel_pasid_alloc_table(struct device *dev) · 100 unmodified lines
101101
102102 might_sleep();
103103 info = dev_iommu_priv_get(dev);
104− if (WARN_ON(!info || !dev_is_pci(dev) || info->pasid_table))
105− return -EINVAL;
104+ if (WARN_ON(!info || !dev_is_pci(dev)))
105+ return -ENODEV;
106+ if (WARN_ON(info->pasid_table))
107+ return -EEXIST;
106108
107109 pasid_table = kzalloc(sizeof(*pasid_table), GFP_KERNEL);
108110 if (!pasid_table)
drivers/iommu/intel/svm.c+57−88
@@ -24,7 +24,7 @@ · 23 unmodified lines
2424 #include "iommu.h"
2525 #include "pasid.h"
2626 #include "perf.h"
27−#include "../iommu-sva-lib.h"
27+#include "../iommu-sva.h"
2828 #include "trace.h"
2929
3030 static irqreturn_t prq_event_thread(int irq, void *d);
@@ -299,19 +299,9 @@ out: · 268 unmodified lines
299299 return 0;
300300 }
301301
302−static int intel_svm_alloc_pasid(struct device *dev, struct mm_struct *mm,
303− unsigned int flags)
304−{
305− ioasid_t max_pasid = dev_is_pci(dev) ?
306− pci_max_pasids(to_pci_dev(dev)) : intel_pasid_max_id;
307−
308− return iommu_sva_alloc_pasid(mm, PASID_MIN, max_pasid - 1);
309−}
310−
311302 static struct iommu_sva *intel_svm_bind_mm(struct intel_iommu *iommu,
312303 struct device *dev,
313− struct mm_struct *mm,
314− unsigned int flags)
304+ struct mm_struct *mm)
315305 {
316306 struct device_domain_info *info = dev_iommu_priv_get(dev);
317307 struct intel_svm_dev *sdev;
@@ -327,22 +317,18 @@ static struct iommu_sva *intel_svm_bind_mm(struct intel_iommu *iommu, · 9 unmodified lines
327317
328318 svm->pasid = mm->pasid;
329319 svm->mm = mm;
330− svm->flags = flags;
331320 INIT_LIST_HEAD_RCU(&svm->devs);
332321
333− if (!(flags & SVM_FLAG_SUPERVISOR_MODE)) {
334− svm->notifier.ops = &intel_mmuops;
335− ret = mmu_notifier_register(&svm->notifier, mm);
336− if (ret) {
337− kfree(svm);
338− return ERR_PTR(ret);
339− }
322+ svm->notifier.ops = &intel_mmuops;
323+ ret = mmu_notifier_register(&svm->notifier, mm);
324+ if (ret) {
325+ kfree(svm);
326+ return ERR_PTR(ret);
340327 }
341328
342329 ret = pasid_private_add(svm->pasid, svm);
343330 if (ret) {
344− if (svm->notifier.ops)
345− mmu_notifier_unregister(&svm->notifier, mm);
331+ mmu_notifier_unregister(&svm->notifier, mm);
346332 kfree(svm);
347333 return ERR_PTR(ret);
348334 }
@@ -377,9 +363,7 @@ static struct iommu_sva *intel_svm_bind_mm(struct intel_iommu *iommu, · 28 unmodified lines
377363 }
378364
379365 /* Setup the pasid table: */
380− sflags = (flags & SVM_FLAG_SUPERVISOR_MODE) ?
381− PASID_FLAG_SUPERVISOR_MODE : 0;
382− sflags |= cpu_feature_enabled(X86_FEATURE_LA57) ? PASID_FLAG_FL5LP : 0;
366+ sflags = cpu_feature_enabled(X86_FEATURE_LA57) ? PASID_FLAG_FL5LP : 0;
383367 ret = intel_pasid_setup_first_level(iommu, dev, mm->pgd, mm->pasid,
384368 FLPT_DEFAULT_DID, sflags);
385369 if (ret)
@@ -393,8 +377,7 @@ free_sdev: · 7 unmodified lines
393377 kfree(sdev);
394378 free_svm:
395379 if (list_empty(&svm->devs)) {
396− if (svm->notifier.ops)
397− mmu_notifier_unregister(&svm->notifier, mm);
380+ mmu_notifier_unregister(&svm->notifier, mm);
398381 pasid_private_remove(mm->pasid);
399382 kfree(svm);
400383 }
@@ -787,67 +770,6 @@ prq_advance: · 386 unmodified lines
787770 return IRQ_RETVAL(handled);
788771 }
789772
790−struct iommu_sva *intel_svm_bind(struct device *dev, struct mm_struct *mm, void *drvdata)
791−{
792− struct intel_iommu *iommu = device_to_iommu(dev, NULL, NULL);
793− unsigned int flags = 0;
794− struct iommu_sva *sva;
795− int ret;
796−
797− if (drvdata)
798− flags = *(unsigned int *)drvdata;
799−
800− if (flags & SVM_FLAG_SUPERVISOR_MODE) {
801− if (!ecap_srs(iommu->ecap)) {
802− dev_err(dev, "%s: Supervisor PASID not supported\n",
803− iommu->name);
804− return ERR_PTR(-EOPNOTSUPP);
805− }
806−
807− if (mm) {
808− dev_err(dev, "%s: Supervisor PASID with user provided mm\n",
809− iommu->name);
810− return ERR_PTR(-EINVAL);
811− }
812−
813− mm = &init_mm;
814− }
815−
816− mutex_lock(&pasid_mutex);
817− ret = intel_svm_alloc_pasid(dev, mm, flags);
818− if (ret) {
819− mutex_unlock(&pasid_mutex);
820− return ERR_PTR(ret);
821− }
822−
823− sva = intel_svm_bind_mm(iommu, dev, mm, flags);
824− mutex_unlock(&pasid_mutex);
825−
826− return sva;
827−}
828−
829−void intel_svm_unbind(struct iommu_sva *sva)
830−{
831− struct intel_svm_dev *sdev = to_intel_svm_dev(sva);
832−
833− mutex_lock(&pasid_mutex);
834− intel_svm_unbind_mm(sdev->dev, sdev->pasid);
835− mutex_unlock(&pasid_mutex);
836−}
837−
838−u32 intel_svm_get_pasid(struct iommu_sva *sva)
839−{
840− struct intel_svm_dev *sdev;
841− u32 pasid;
842−
843− mutex_lock(&pasid_mutex);
844− sdev = to_intel_svm_dev(sva);
845− pasid = sdev->pasid;
846− mutex_unlock(&pasid_mutex);
847−
848− return pasid;
849−}
850−
851773 int intel_svm_page_response(struct device *dev,
852774 struct iommu_fault_event *evt,
853775 struct iommu_page_response *msg)
@@ -918,3 +840,50 @@ int intel_svm_page_response(struct device *dev, · 64 unmodified lines
918840 out:
919841 return ret;
920842 }
843+
844+void intel_svm_remove_dev_pasid(struct device *dev, ioasid_t pasid)
845+{
846+ mutex_lock(&pasid_mutex);
847+ intel_svm_unbind_mm(dev, pasid);
848+ mutex_unlock(&pasid_mutex);
849+}
850+
851+static int intel_svm_set_dev_pasid(struct iommu_domain *domain,
852+ struct device *dev, ioasid_t pasid)
853+{
854+ struct device_domain_info *info = dev_iommu_priv_get(dev);
855+ struct intel_iommu *iommu = info->iommu;
856+ struct mm_struct *mm = domain->mm;
857+ struct iommu_sva *sva;
858+ int ret = 0;
859+
860+ mutex_lock(&pasid_mutex);
861+ sva = intel_svm_bind_mm(iommu, dev, mm);
862+ if (IS_ERR(sva))
863+ ret = PTR_ERR(sva);
864+ mutex_unlock(&pasid_mutex);
865+
866+ return ret;
867+}
868+
869+static void intel_svm_domain_free(struct iommu_domain *domain)
870+{
871+ kfree(to_dmar_domain(domain));
872+}
873+
874+static const struct iommu_domain_ops intel_svm_domain_ops = {
875+ .set_dev_pasid = intel_svm_set_dev_pasid,
876+ .free = intel_svm_domain_free
877+};
878+
879+struct iommu_domain *intel_svm_domain_alloc(void)
880+{
881+ struct dmar_domain *domain;
882+
883+ domain = kzalloc(sizeof(*domain), GFP_KERNEL);
884+ if (!domain)
885+ return NULL;
886+ domain->domain.ops = &intel_svm_domain_ops;
887+
888+ return &domain->domain;
889+}
drivers/iommu/io-pgfault.c+17−60
@@ -11,7 +11,7 @@ · 10 unmodified lines
1111 #include <linux/slab.h>
1212 #include <linux/workqueue.h>
1313
14−#include "iommu-sva-lib.h"
14+#include "iommu-sva.h"
1515
1616 /**
1717 * struct iopf_queue - IO Page Fault queue
@@ -69,69 +69,18 @@ static int iopf_complete_group(struct device *dev, struct iopf_fault *iopf, · 51 unmodified lines
6969 return iommu_page_response(dev, &resp);
7070 }
7171
72−static enum iommu_page_response_code
73−iopf_handle_single(struct iopf_fault *iopf)
74−{
75− vm_fault_t ret;
76− struct mm_struct *mm;
77− struct vm_area_struct *vma;
78− unsigned int access_flags = 0;
79− unsigned int fault_flags = FAULT_FLAG_REMOTE;
80− struct iommu_fault_page_request *prm = &iopf->fault.prm;
81− enum iommu_page_response_code status = IOMMU_PAGE_RESP_INVALID;
82−
83− if (!(prm->flags & IOMMU_FAULT_PAGE_REQUEST_PASID_VALID))
84− return status;
85−
86− mm = iommu_sva_find(prm->pasid);
87− if (IS_ERR_OR_NULL(mm))
88− return status;
89−
90− mmap_read_lock(mm);
91−
92− vma = find_extend_vma(mm, prm->addr);
93− if (!vma)
94− /* Unmapped area */
95− goto out_put_mm;
96−
97− if (prm->perm & IOMMU_FAULT_PERM_READ)
98− access_flags |= VM_READ;
99−
100− if (prm->perm & IOMMU_FAULT_PERM_WRITE) {
101− access_flags |= VM_WRITE;
102− fault_flags |= FAULT_FLAG_WRITE;
103− }
104−
105− if (prm->perm & IOMMU_FAULT_PERM_EXEC) {
106− access_flags |= VM_EXEC;
107− fault_flags |= FAULT_FLAG_INSTRUCTION;
108− }
109−
110− if (!(prm->perm & IOMMU_FAULT_PERM_PRIV))
111− fault_flags |= FAULT_FLAG_USER;
112−
113− if (access_flags & ~vma->vm_flags)
114− /* Access fault */
115− goto out_put_mm;
116−
117− ret = handle_mm_fault(vma, prm->addr, fault_flags, NULL);
118− status = ret & VM_FAULT_ERROR ? IOMMU_PAGE_RESP_INVALID :
119− IOMMU_PAGE_RESP_SUCCESS;
120−
121−out_put_mm:
122− mmap_read_unlock(mm);
123− mmput(mm);
124−
125− return status;
126−}
127−
128−static void iopf_handle_group(struct work_struct *work)
72+static void iopf_handler(struct work_struct *work)
12973 {
13074 struct iopf_group *group;
75+ struct iommu_domain *domain;
13176 struct iopf_fault *iopf, *next;
13277 enum iommu_page_response_code status = IOMMU_PAGE_RESP_SUCCESS;
13378
13479 group = container_of(work, struct iopf_group, work);
80+ domain = iommu_get_domain_for_dev_pasid(group->dev,
81+ group->last_fault.fault.prm.pasid, 0);
82+ if (!domain || !domain->iopf_handler)
83+ status = IOMMU_PAGE_RESP_INVALID;
13584
13685 list_for_each_entry_safe(iopf, next, &group->faults, list) {
13786 /*
@@ -139,7 +88,8 @@ static void iopf_handle_group(struct work_struct *work) · 1 unmodified line
13988 * faults in the group if there is an error.
14089 */
14190 if (status == IOMMU_PAGE_RESP_SUCCESS)
142− status = iopf_handle_single(iopf);
91+ status = domain->iopf_handler(&iopf->fault,
92+ domain->fault_data);
14393
14494 if (!(iopf->fault.prm.flags &
14595 IOMMU_FAULT_PAGE_REQUEST_LAST_PAGE))
@@ -181,6 +131,13 @@ static void iopf_handle_group(struct work_struct *work) · 35 unmodified lines
181131 * request completes, outstanding faults will have been dealt with by the time
182132 * the PASID is freed.
183133 *
134+ * Any valid page fault will be eventually routed to an iommu domain and the
135+ * page fault handler installed there will get called. The users of this
136+ * handling framework should guarantee that the iommu domain could only be
137+ * freed after the device has stopped generating page faults (or the iommu
138+ * hardware has been set to block the page faults) and the pending page faults
139+ * have been flushed.
140+ *
184141 * Return: 0 on success and <0 on error.
185142 */
186143 int iommu_queue_iopf(struct iommu_fault *fault, void *cookie)
@@ -235,7 +192,7 @@ int iommu_queue_iopf(struct iommu_fault *fault, void *cookie) · 48 unmodified lines
235192 group->last_fault.fault = *fault;
236193 INIT_LIST_HEAD(&group->faults);
237194 list_add(&group->last_fault.list, &group->faults);
238− INIT_WORK(&group->work, iopf_handle_group);
195+ INIT_WORK(&group->work, iopf_handler);
239196
240197 /* See if we have partial faults for this group */
241198 list_for_each_entry_safe(iopf, next, &iopf_param->partial, list) {
drivers/iommu/iommu-sva-lib.c+0−71
@@ -1,71 +0,0 @@
1−// SPDX-License-Identifier: GPL-2.0
2−/*
3− * Helpers for IOMMU drivers implementing SVA
4− */
5−#include <linux/mutex.h>
6−#include <linux/sched/mm.h>
7−
8−#include "iommu-sva-lib.h"
9−
10−static DEFINE_MUTEX(iommu_sva_lock);
11−static DECLARE_IOASID_SET(iommu_sva_pasid);
12−
13−/**
14− * iommu_sva_alloc_pasid - Allocate a PASID for the mm
15− * @mm: the mm
16− * @min: minimum PASID value (inclusive)
17− * @max: maximum PASID value (inclusive)
18− *
19− * Try to allocate a PASID for this mm, or take a reference to the existing one
20− * provided it fits within the [@min, @max] range. On success the PASID is
21− * available in mm->pasid and will be available for the lifetime of the mm.
22− *
23− * Returns 0 on success and < 0 on error.
24− */
25−int iommu_sva_alloc_pasid(struct mm_struct *mm, ioasid_t min, ioasid_t max)
26−{
27− int ret = 0;
28− ioasid_t pasid;
29−
30− if (min == INVALID_IOASID || max == INVALID_IOASID ||
31− min == 0 || max < min)
32− return -EINVAL;
33−
34− mutex_lock(&iommu_sva_lock);
35− /* Is a PASID already associated with this mm? */
36− if (pasid_valid(mm->pasid)) {
37− if (mm->pasid < min || mm->pasid >= max)
38− ret = -EOVERFLOW;
39− goto out;
40− }
41−
42− pasid = ioasid_alloc(&iommu_sva_pasid, min, max, mm);
43− if (!pasid_valid(pasid))
44− ret = -ENOMEM;
45− else
46− mm_pasid_set(mm, pasid);
47−out:
48− mutex_unlock(&iommu_sva_lock);
49− return ret;
50−}
51−EXPORT_SYMBOL_GPL(iommu_sva_alloc_pasid);
52−
53−/* ioasid_find getter() requires a void * argument */
54−static bool __mmget_not_zero(void *mm)
55−{
56− return mmget_not_zero(mm);
57−}
58−
59−/**
60− * iommu_sva_find() - Find mm associated to the given PASID
61− * @pasid: Process Address Space ID assigned to the mm
62− *
63− * On success a reference to the mm is taken, and must be released with mmput().
64− *
65− * Returns the mm corresponding to this PASID, or an error if not found.
66− */
67−struct mm_struct *iommu_sva_find(ioasid_t pasid)
68−{
69− return ioasid_find(&iommu_sva_pasid, pasid, __mmget_not_zero);
70−}
71−EXPORT_SYMBOL_GPL(iommu_sva_find);
drivers/iommu/iommu-sva.c+240−0
@@ -0,0 +1,240 @@
1+// SPDX-License-Identifier: GPL-2.0
2+/*
3+ * Helpers for IOMMU drivers implementing SVA
4+ */
5+#include <linux/mutex.h>
6+#include <linux/sched/mm.h>
7+#include <linux/iommu.h>
8+
9+#include "iommu-sva.h"
10+
11+static DEFINE_MUTEX(iommu_sva_lock);
12+static DECLARE_IOASID_SET(iommu_sva_pasid);
13+
14+/**
15+ * iommu_sva_alloc_pasid - Allocate a PASID for the mm
16+ * @mm: the mm
17+ * @min: minimum PASID value (inclusive)
18+ * @max: maximum PASID value (inclusive)
19+ *
20+ * Try to allocate a PASID for this mm, or take a reference to the existing one
21+ * provided it fits within the [@min, @max] range. On success the PASID is
22+ * available in mm->pasid and will be available for the lifetime of the mm.
23+ *
24+ * Returns 0 on success and < 0 on error.
25+ */
26+int iommu_sva_alloc_pasid(struct mm_struct *mm, ioasid_t min, ioasid_t max)
27+{
28+ int ret = 0;
29+ ioasid_t pasid;
30+
31+ if (min == INVALID_IOASID || max == INVALID_IOASID ||
32+ min == 0 || max < min)
33+ return -EINVAL;
34+
35+ mutex_lock(&iommu_sva_lock);
36+ /* Is a PASID already associated with this mm? */
37+ if (pasid_valid(mm->pasid)) {
38+ if (mm->pasid < min || mm->pasid >= max)
39+ ret = -EOVERFLOW;
40+ goto out;
41+ }
42+
43+ pasid = ioasid_alloc(&iommu_sva_pasid, min, max, mm);
44+ if (!pasid_valid(pasid))
45+ ret = -ENOMEM;
46+ else
47+ mm_pasid_set(mm, pasid);
48+out:
49+ mutex_unlock(&iommu_sva_lock);
50+ return ret;
51+}
52+EXPORT_SYMBOL_GPL(iommu_sva_alloc_pasid);
53+
54+/* ioasid_find getter() requires a void * argument */
55+static bool __mmget_not_zero(void *mm)
56+{
57+ return mmget_not_zero(mm);
58+}
59+
60+/**
61+ * iommu_sva_find() - Find mm associated to the given PASID
62+ * @pasid: Process Address Space ID assigned to the mm
63+ *
64+ * On success a reference to the mm is taken, and must be released with mmput().
65+ *
66+ * Returns the mm corresponding to this PASID, or an error if not found.
67+ */
68+struct mm_struct *iommu_sva_find(ioasid_t pasid)
69+{
70+ return ioasid_find(&iommu_sva_pasid, pasid, __mmget_not_zero);
71+}
72+EXPORT_SYMBOL_GPL(iommu_sva_find);
73+
74+/**
75+ * iommu_sva_bind_device() - Bind a process address space to a device
76+ * @dev: the device
77+ * @mm: the mm to bind, caller must hold a reference to mm_users
78+ *
79+ * Create a bond between device and address space, allowing the device to
80+ * access the mm using the PASID returned by iommu_sva_get_pasid(). If a
81+ * bond already exists between @device and @mm, an additional internal
82+ * reference is taken. Caller must call iommu_sva_unbind_device()
83+ * to release each reference.
84+ *
85+ * iommu_dev_enable_feature(dev, IOMMU_DEV_FEAT_SVA) must be called first, to
86+ * initialize the required SVA features.
87+ *
88+ * On error, returns an ERR_PTR value.
89+ */
90+struct iommu_sva *iommu_sva_bind_device(struct device *dev, struct mm_struct *mm)
91+{
92+ struct iommu_domain *domain;
93+ struct iommu_sva *handle;
94+ ioasid_t max_pasids;
95+ int ret;
96+
97+ max_pasids = dev->iommu->max_pasids;
98+ if (!max_pasids)
99+ return ERR_PTR(-EOPNOTSUPP);
100+
101+ /* Allocate mm->pasid if necessary. */
102+ ret = iommu_sva_alloc_pasid(mm, 1, max_pasids - 1);
103+ if (ret)
104+ return ERR_PTR(ret);
105+
106+ handle = kzalloc(sizeof(*handle), GFP_KERNEL);
107+ if (!handle)
108+ return ERR_PTR(-ENOMEM);
109+
110+ mutex_lock(&iommu_sva_lock);
111+ /* Search for an existing domain. */
112+ domain = iommu_get_domain_for_dev_pasid(dev, mm->pasid,
113+ IOMMU_DOMAIN_SVA);
114+ if (IS_ERR(domain)) {
115+ ret = PTR_ERR(domain);
116+ goto out_unlock;
117+ }
118+
119+ if (domain) {
120+ domain->users++;
121+ goto out;
122+ }
123+
124+ /* Allocate a new domain and set it on device pasid. */
125+ domain = iommu_sva_domain_alloc(dev, mm);
126+ if (!domain) {
127+ ret = -ENOMEM;
128+ goto out_unlock;
129+ }
130+
131+ ret = iommu_attach_device_pasid(domain, dev, mm->pasid);
132+ if (ret)
133+ goto out_free_domain;
134+ domain->users = 1;
135+out:
136+ mutex_unlock(&iommu_sva_lock);
137+ handle->dev = dev;
138+ handle->domain = domain;
139+
140+ return handle;
141+
142+out_free_domain:
143+ iommu_domain_free(domain);
144+out_unlock:
145+ mutex_unlock(&iommu_sva_lock);
146+ kfree(handle);
147+
148+ return ERR_PTR(ret);
149+}
150+EXPORT_SYMBOL_GPL(iommu_sva_bind_device);
151+
152+/**
153+ * iommu_sva_unbind_device() - Remove a bond created with iommu_sva_bind_device
154+ * @handle: the handle returned by iommu_sva_bind_device()
155+ *
156+ * Put reference to a bond between device and address space. The device should
157+ * not be issuing any more transaction for this PASID. All outstanding page
158+ * requests for this PASID must have been flushed to the IOMMU.
159+ */
160+void iommu_sva_unbind_device(struct iommu_sva *handle)
161+{
162+ struct iommu_domain *domain = handle->domain;
163+ ioasid_t pasid = domain->mm->pasid;
164+ struct device *dev = handle->dev;
165+
166+ mutex_lock(&iommu_sva_lock);
167+ if (--domain->users == 0) {
168+ iommu_detach_device_pasid(domain, dev, pasid);
169+ iommu_domain_free(domain);
170+ }
171+ mutex_unlock(&iommu_sva_lock);
172+ kfree(handle);
173+}
174+EXPORT_SYMBOL_GPL(iommu_sva_unbind_device);
175+
176+u32 iommu_sva_get_pasid(struct iommu_sva *handle)
177+{
178+ struct iommu_domain *domain = handle->domain;
179+
180+ return domain->mm->pasid;
181+}
182+EXPORT_SYMBOL_GPL(iommu_sva_get_pasid);
183+
184+/*
185+ * I/O page fault handler for SVA
186+ */
187+enum iommu_page_response_code
188+iommu_sva_handle_iopf(struct iommu_fault *fault, void *data)
189+{
190+ vm_fault_t ret;
191+ struct vm_area_struct *vma;
192+ struct mm_struct *mm = data;
193+ unsigned int access_flags = 0;
194+ unsigned int fault_flags = FAULT_FLAG_REMOTE;
195+ struct iommu_fault_page_request *prm = &fault->prm;
196+ enum iommu_page_response_code status = IOMMU_PAGE_RESP_INVALID;
197+
198+ if (!(prm->flags & IOMMU_FAULT_PAGE_REQUEST_PASID_VALID))
199+ return status;
200+
201+ if (!mmget_not_zero(mm))
202+ return status;
203+
204+ mmap_read_lock(mm);
205+
206+ vma = find_extend_vma(mm, prm->addr);
207+ if (!vma)
208+ /* Unmapped area */
209+ goto out_put_mm;
210+
211+ if (prm->perm & IOMMU_FAULT_PERM_READ)
212+ access_flags |= VM_READ;
213+
214+ if (prm->perm & IOMMU_FAULT_PERM_WRITE) {
215+ access_flags |= VM_WRITE;
216+ fault_flags |= FAULT_FLAG_WRITE;
217+ }
218+
219+ if (prm->perm & IOMMU_FAULT_PERM_EXEC) {
220+ access_flags |= VM_EXEC;
221+ fault_flags |= FAULT_FLAG_INSTRUCTION;
222+ }
223+
224+ if (!(prm->perm & IOMMU_FAULT_PERM_PRIV))
225+ fault_flags |= FAULT_FLAG_USER;
226+
227+ if (access_flags & ~vma->vm_flags)
228+ /* Access fault */
229+ goto out_put_mm;
230+
231+ ret = handle_mm_fault(vma, prm->addr, fault_flags, NULL);
232+ status = ret & VM_FAULT_ERROR ? IOMMU_PAGE_RESP_INVALID :
233+ IOMMU_PAGE_RESP_SUCCESS;
234+
235+out_put_mm:
236+ mmap_read_unlock(mm);
237+ mmput(mm);
238+
239+ return status;
240+}
drivers/iommu/iommu-sva.hrenamed from drivers/iommu/iommu-sva-lib.h+11−3
@@ -2,8 +2,8 @@ · 1 unmodified line
22 /*
33 * SVA library for IOMMU drivers
44 */
5−#ifndef _IOMMU_SVA_LIB_H
6−#define _IOMMU_SVA_LIB_H
5+#ifndef _IOMMU_SVA_H
6+#define _IOMMU_SVA_H
77
88 #include <linux/ioasid.h>
99 #include <linux/mm_types.h>
@@ -26,6 +26,8 @@ int iopf_queue_flush_dev(struct device *dev); · 16 unmodified lines
2626 struct iopf_queue *iopf_queue_alloc(const char *name);
2727 void iopf_queue_free(struct iopf_queue *queue);
2828 int iopf_queue_discard_partial(struct iopf_queue *queue);
29+enum iommu_page_response_code
30+iommu_sva_handle_iopf(struct iommu_fault *fault, void *data);
2931
3032 #else /* CONFIG_IOMMU_SVA */
3133 static inline int iommu_queue_iopf(struct iommu_fault *fault, void *cookie)
@@ -63,5 +65,11 @@ static inline int iopf_queue_discard_partial(struct iopf_queue *queue) · 31 unmodified lines
6365 {
6466 return -ENODEV;
6567 }
68+
69+static inline enum iommu_page_response_code
70+iommu_sva_handle_iopf(struct iommu_fault *fault, void *data)
71+{
72+ return IOMMU_PAGE_RESP_INVALID;
73+}
6674 #endif /* CONFIG_IOMMU_SVA */
67−#endif /* _IOMMU_SVA_LIB_H */
75+#endif /* _IOMMU_SVA_H */
drivers/iommu/iommu.c+297−119
@@ -21,6 +21,7 @@ · 20 unmodified lines
2121 #include <linux/idr.h>
2222 #include <linux/err.h>
2323 #include <linux/pci.h>
24+#include <linux/pci-ats.h>
2425 #include <linux/bitops.h>
2526 #include <linux/platform_device.h>
2627 #include <linux/property.h>
@@ -28,9 +29,12 @@ · 1 unmodified line
2829 #include <linux/module.h>
2930 #include <linux/cc_platform.h>
3031 #include <trace/events/iommu.h>
32+#include <linux/sched/mm.h>
3133
3234 #include "dma-iommu.h"
3335
36+#include "iommu-sva.h"
37+
3438 static struct kset *iommu_group_kset;
3539 static DEFINE_IDA(iommu_group_ida);
3640
@@ -42,6 +46,7 @@ struct iommu_group { · 5 unmodified lines
4246 struct kobject kobj;
4347 struct kobject *devices_kobj;
4448 struct list_head devices;
49+ struct xarray pasid_array;
4550 struct mutex mutex;
4651 void *iommu_data;
4752 void (*iommu_data_release)(void *iommu_data);
@@ -278,6 +283,24 @@ static void dev_iommu_free(struct device *dev) · 230 unmodified lines
278283 kfree(param);
279284 }
280285
286+static u32 dev_iommu_get_max_pasids(struct device *dev)
287+{
288+ u32 max_pasids = 0, bits = 0;
289+ int ret;
290+
291+ if (dev_is_pci(dev)) {
292+ ret = pci_max_pasids(to_pci_dev(dev));
293+ if (ret > 0)
294+ max_pasids = ret;
295+ } else {
296+ ret = device_property_read_u32(dev, "pasid-num-bits", &bits);
297+ if (!ret)
298+ max_pasids = 1UL << bits;
299+ }
300+
301+ return min_t(u32, max_pasids, dev->iommu->iommu_dev->max_pasids);
302+}
303+
281304 static int __iommu_probe_device(struct device *dev, struct list_head *group_list)
282305 {
283306 const struct iommu_ops *ops = dev->bus->iommu_ops;
@@ -303,6 +326,7 @@ static int __iommu_probe_device(struct device *dev, struct list_head *group_list · 19 unmodified lines
303326 }
304327
305328 dev->iommu->iommu_dev = iommu_dev;
329+ dev->iommu->max_pasids = dev_iommu_get_max_pasids(dev);
306330
307331 group = iommu_group_get_for_dev(dev);
308332 if (IS_ERR(group)) {
@@ -703,6 +727,7 @@ struct iommu_group *iommu_group_alloc(void) · 394 unmodified lines
703727 mutex_init(&group->mutex);
704728 INIT_LIST_HEAD(&group->devices);
705729 INIT_LIST_HEAD(&group->entry);
730+ xa_init(&group->pasid_array);
706731
707732 ret = ida_alloc(&iommu_group_ida, GFP_KERNEL);
708733 if (ret < 0) {
@@ -1912,6 +1937,8 @@ EXPORT_SYMBOL_GPL(iommu_domain_alloc); · 1203 unmodified lines
19121937
19131938 void iommu_domain_free(struct iommu_domain *domain)
19141939 {
1940+ if (domain->type == IOMMU_DOMAIN_SVA)
1941+ mmdrop(domain->mm);
19151942 iommu_put_dma_cookie(domain);
19161943 domain->ops->free(domain);
19171944 }
@@ -1949,6 +1976,18 @@ static int __iommu_attach_device(struct iommu_domain *domain, · 31 unmodified lines
19491976 return ret;
19501977 }
19511978
1979+/**
1980+ * iommu_attach_device - Attach an IOMMU domain to a device
1981+ * @domain: IOMMU domain to attach
1982+ * @dev: Device that will be attached
1983+ *
1984+ * Returns 0 on success and error code on failure
1985+ *
1986+ * Note that EINVAL can be treated as a soft failure, indicating
1987+ * that certain configuration of the domain is incompatible with
1988+ * the device. In this case attaching a different domain to the
1989+ * device may succeed.
1990+ */
19521991 int iommu_attach_device(struct iommu_domain *domain, struct device *dev)
19531992 {
19541993 struct iommu_group *group;
@@ -2075,6 +2114,18 @@ static int __iommu_attach_group(struct iommu_domain *domain, · 120 unmodified lines
20752114 return ret;
20762115 }
20772116
2117+/**
2118+ * iommu_attach_group - Attach an IOMMU domain to an IOMMU group
2119+ * @domain: IOMMU domain to attach
2120+ * @group: IOMMU group that will be attached
2121+ *
2122+ * Returns 0 on success and error code on failure
2123+ *
2124+ * Note that EINVAL can be treated as a soft failure, indicating
2125+ * that certain configuration of the domain is incompatible with
2126+ * the group. In this case attaching a different domain to the
2127+ * group may succeed.
2128+ */
20782129 int iommu_attach_group(struct iommu_domain *domain, struct iommu_group *group)
20792130 {
20802131 int ret;
@@ -2726,98 +2777,6 @@ int iommu_dev_disable_feature(struct device *dev, enum iommu_dev_features feat) · 645 unmodified lines
27262777 }
27272778 EXPORT_SYMBOL_GPL(iommu_dev_disable_feature);
27282779
2729−/**
2730− * iommu_sva_bind_device() - Bind a process address space to a device
2731− * @dev: the device
2732− * @mm: the mm to bind, caller must hold a reference to it
2733− * @drvdata: opaque data pointer to pass to bind callback
2734− *
2735− * Create a bond between device and address space, allowing the device to access
2736− * the mm using the returned PASID. If a bond already exists between @device and
2737− * @mm, it is returned and an additional reference is taken. Caller must call
2738− * iommu_sva_unbind_device() to release each reference.
2739− *
2740− * iommu_dev_enable_feature(dev, IOMMU_DEV_FEAT_SVA) must be called first, to
2741− * initialize the required SVA features.
2742− *
2743− * On error, returns an ERR_PTR value.
2744− */
2745−struct iommu_sva *
2746−iommu_sva_bind_device(struct device *dev, struct mm_struct *mm, void *drvdata)
2747−{
2748− struct iommu_group *group;
2749− struct iommu_sva *handle = ERR_PTR(-EINVAL);
2750− const struct iommu_ops *ops = dev_iommu_ops(dev);
2751−
2752− if (!ops->sva_bind)
2753− return ERR_PTR(-ENODEV);
2754−
2755− group = iommu_group_get(dev);
2756− if (!group)
2757− return ERR_PTR(-ENODEV);
2758−
2759− /* Ensure device count and domain don't change while we're binding */
2760− mutex_lock(&group->mutex);
2761−
2762− /*
2763− * To keep things simple, SVA currently doesn't support IOMMU groups
2764− * with more than one device. Existing SVA-capable systems are not
2765− * affected by the problems that required IOMMU groups (lack of ACS
2766− * isolation, device ID aliasing and other hardware issues).
2767− */
2768− if (iommu_group_device_count(group) != 1)
2769− goto out_unlock;
2770−
2771− handle = ops->sva_bind(dev, mm, drvdata);
2772−
2773−out_unlock:
2774− mutex_unlock(&group->mutex);
2775− iommu_group_put(group);
2776−
2777− return handle;
2778−}
2779−EXPORT_SYMBOL_GPL(iommu_sva_bind_device);
2780−
2781−/**
2782− * iommu_sva_unbind_device() - Remove a bond created with iommu_sva_bind_device
2783− * @handle: the handle returned by iommu_sva_bind_device()
2784− *
2785− * Put reference to a bond between device and address space. The device should
2786− * not be issuing any more transaction for this PASID. All outstanding page
2787− * requests for this PASID must have been flushed to the IOMMU.
2788− */
2789−void iommu_sva_unbind_device(struct iommu_sva *handle)
2790−{
2791− struct iommu_group *group;
2792− struct device *dev = handle->dev;
2793− const struct iommu_ops *ops = dev_iommu_ops(dev);
2794−
2795− if (!ops->sva_unbind)
2796− return;
2797−
2798− group = iommu_group_get(dev);
2799− if (!group)
2800− return;
2801−
2802− mutex_lock(&group->mutex);
2803− ops->sva_unbind(handle);
2804− mutex_unlock(&group->mutex);
2805−
2806− iommu_group_put(group);
2807−}
2808−EXPORT_SYMBOL_GPL(iommu_sva_unbind_device);
2809−
2810−u32 iommu_sva_get_pasid(struct iommu_sva *handle)
2811−{
2812− const struct iommu_ops *ops = dev_iommu_ops(handle->dev);
2813−
2814− if (!ops->sva_get_pasid)
2815− return IOMMU_PASID_INVALID;
2816−
2817− return ops->sva_get_pasid(handle);
2818−}
2819−EXPORT_SYMBOL_GPL(iommu_sva_get_pasid);
2820−
28212780 /*
28222781 * Changes the default domain of an iommu group that has *only* one device
28232782 *
@@ -3087,7 +3046,8 @@ int iommu_device_use_default_domain(struct device *dev) · 263 unmodified lines
30873046
30883047 mutex_lock(&group->mutex);
30893048 if (group->owner_cnt) {
3090− if (group->owner || !iommu_is_default_domain(group)) {
3049+ if (group->owner || !iommu_is_default_domain(group) ||
3050+ !xa_empty(&group->pasid_array)) {
30913051 ret = -EBUSY;
30923052 goto unlock_out;
30933053 }
@@ -3118,7 +3078,7 @@ void iommu_device_unuse_default_domain(struct device *dev) · 24 unmodified lines
31183078 return;
31193079
31203080 mutex_lock(&group->mutex);
3121− if (!WARN_ON(!group->owner_cnt))
3081+ if (!WARN_ON(!group->owner_cnt || !xa_empty(&group->pasid_array)))
31223082 group->owner_cnt--;
31233083
31243084 mutex_unlock(&group->mutex);
@@ -3148,40 +3108,49 @@ static int __iommu_group_alloc_blocking_domain(struct iommu_group *group) · 23 unmodified lines
31483108 return 0;
31493109 }
31503110
3111+static int __iommu_take_dma_ownership(struct iommu_group *group, void *owner)
3112+{
3113+ int ret;
3114+
3115+ if ((group->domain && group->domain != group->default_domain) ||
3116+ !xa_empty(&group->pasid_array))
3117+ return -EBUSY;
3118+
3119+ ret = __iommu_group_alloc_blocking_domain(group);
3120+ if (ret)
3121+ return ret;
3122+ ret = __iommu_group_set_domain(group, group->blocking_domain);
3123+ if (ret)
3124+ return ret;
3125+
3126+ group->owner = owner;
3127+ group->owner_cnt++;
3128+ return 0;
3129+}
3130+
31513131 /**
31523132 * iommu_group_claim_dma_owner() - Set DMA ownership of a group
31533133 * @group: The group.
31543134 * @owner: Caller specified pointer. Used for exclusive ownership.
31553135 *
3156− * This is to support backward compatibility for vfio which manages
3157− * the dma ownership in iommu_group level. New invocations on this
3158− * interface should be prohibited.
3136+ * This is to support backward compatibility for vfio which manages the dma
3137+ * ownership in iommu_group level. New invocations on this interface should be
3138+ * prohibited. Only a single owner may exist for a group.
31593139 */
31603140 int iommu_group_claim_dma_owner(struct iommu_group *group, void *owner)
31613141 {
31623142 int ret = 0;
31633143
3144+ if (WARN_ON(!owner))
3145+ return -EINVAL;
3146+
31643147 mutex_lock(&group->mutex);
31653148 if (group->owner_cnt) {
31663149 ret = -EPERM;
31673150 goto unlock_out;
3168− } else {
3169− if (group->domain && group->domain != group->default_domain) {
3170− ret = -EBUSY;
3171− goto unlock_out;
3172− }
3173−
3174− ret = __iommu_group_alloc_blocking_domain(group);
3175− if (ret)
3176− goto unlock_out;
3177−
3178− ret = __iommu_group_set_domain(group, group->blocking_domain);
3179− if (ret)
3180− goto unlock_out;
3181− group->owner = owner;
31823151 }
31833152
3184− group->owner_cnt++;
3153+ ret = __iommu_take_dma_ownership(group, owner);
31853154 unlock_out:
31863155 mutex_unlock(&group->mutex);
31873156
@@ -3190,29 +3159,91 @@ unlock_out: · 2 unmodified lines
31903159 EXPORT_SYMBOL_GPL(iommu_group_claim_dma_owner);
31913160
31923161 /**
3193− * iommu_group_release_dma_owner() - Release DMA ownership of a group
3194− * @group: The group.
3162+ * iommu_device_claim_dma_owner() - Set DMA ownership of a device
3163+ * @dev: The device.
3164+ * @owner: Caller specified pointer. Used for exclusive ownership.
31953165 *
3196− * Release the DMA ownership claimed by iommu_group_claim_dma_owner().
3166+ * Claim the DMA ownership of a device. Multiple devices in the same group may
3167+ * concurrently claim ownership if they present the same owner value. Returns 0
3168+ * on success and error code on failure
31973169 */
3198−void iommu_group_release_dma_owner(struct iommu_group *group)
3170+int iommu_device_claim_dma_owner(struct device *dev, void *owner)
31993171 {
3200− int ret;
3172+ struct iommu_group *group = iommu_group_get(dev);
3173+ int ret = 0;
3174+
3175+ if (!group)
3176+ return -ENODEV;
3177+ if (WARN_ON(!owner))
3178+ return -EINVAL;
32013179
32023180 mutex_lock(&group->mutex);
3203− if (WARN_ON(!group->owner_cnt || !group->owner))
3181+ if (group->owner_cnt) {
3182+ if (group->owner != owner) {
3183+ ret = -EPERM;
3184+ goto unlock_out;
3185+ }
3186+ group->owner_cnt++;
32043187 goto unlock_out;
3188+ }
3189+
3190+ ret = __iommu_take_dma_ownership(group, owner);
3191+unlock_out:
3192+ mutex_unlock(&group->mutex);
3193+ iommu_group_put(group);
3194+
3195+ return ret;
3196+}
3197+EXPORT_SYMBOL_GPL(iommu_device_claim_dma_owner);
3198+
3199+static void __iommu_release_dma_ownership(struct iommu_group *group)
3200+{
3201+ int ret;
3202+
3203+ if (WARN_ON(!group->owner_cnt || !group->owner ||
3204+ !xa_empty(&group->pasid_array)))
3205+ return;
32053206
32063207 group->owner_cnt = 0;
32073208 group->owner = NULL;
32083209 ret = __iommu_group_set_domain(group, group->default_domain);
32093210 WARN(ret, "iommu driver failed to attach the default domain");
3211+}
32103212
3211−unlock_out:
3213+/**
3214+ * iommu_group_release_dma_owner() - Release DMA ownership of a group
3215+ * @dev: The device
3216+ *
3217+ * Release the DMA ownership claimed by iommu_group_claim_dma_owner().
3218+ */
3219+void iommu_group_release_dma_owner(struct iommu_group *group)
3220+{
3221+ mutex_lock(&group->mutex);
3222+ __iommu_release_dma_ownership(group);
32123223 mutex_unlock(&group->mutex);
32133224 }
32143225 EXPORT_SYMBOL_GPL(iommu_group_release_dma_owner);
32153226
3227+/**
3228+ * iommu_device_release_dma_owner() - Release DMA ownership of a device
3229+ * @group: The device.
3230+ *
3231+ * Release the DMA ownership claimed by iommu_device_claim_dma_owner().
3232+ */
3233+void iommu_device_release_dma_owner(struct device *dev)
3234+{
3235+ struct iommu_group *group = iommu_group_get(dev);
3236+
3237+ mutex_lock(&group->mutex);
3238+ if (group->owner_cnt > 1)
3239+ group->owner_cnt--;
3240+ else
3241+ __iommu_release_dma_ownership(group);
3242+ mutex_unlock(&group->mutex);
3243+ iommu_group_put(group);
3244+}
3245+EXPORT_SYMBOL_GPL(iommu_device_release_dma_owner);
3246+
32163247 /**
32173248 * iommu_group_dma_owner_claimed() - Query group dma ownership status
32183249 * @group: The group.
@@ -3231,3 +3262,150 @@ bool iommu_group_dma_owner_claimed(struct iommu_group *group) · 12 unmodified lines
32313262 return user;
32323263 }
32333264 EXPORT_SYMBOL_GPL(iommu_group_dma_owner_claimed);
3265+
3266+static int __iommu_set_group_pasid(struct iommu_domain *domain,
3267+ struct iommu_group *group, ioasid_t pasid)
3268+{
3269+ struct group_device *device;
3270+ int ret = 0;
3271+
3272+ list_for_each_entry(device, &group->devices, list) {
3273+ ret = domain->ops->set_dev_pasid(domain, device->dev, pasid);
3274+ if (ret)
3275+ break;
3276+ }
3277+
3278+ return ret;
3279+}
3280+
3281+static void __iommu_remove_group_pasid(struct iommu_group *group,
3282+ ioasid_t pasid)
3283+{
3284+ struct group_device *device;
3285+ const struct iommu_ops *ops;
3286+
3287+ list_for_each_entry(device, &group->devices, list) {
3288+ ops = dev_iommu_ops(device->dev);
3289+ ops->remove_dev_pasid(device->dev, pasid);
3290+ }
3291+}
3292+
3293+/*
3294+ * iommu_attach_device_pasid() - Attach a domain to pasid of device
3295+ * @domain: the iommu domain.
3296+ * @dev: the attached device.
3297+ * @pasid: the pasid of the device.
3298+ *
3299+ * Return: 0 on success, or an error.
3300+ */
3301+int iommu_attach_device_pasid(struct iommu_domain *domain,
3302+ struct device *dev, ioasid_t pasid)
3303+{
3304+ struct iommu_group *group;
3305+ void *curr;
3306+ int ret;
3307+
3308+ if (!domain->ops->set_dev_pasid)
3309+ return -EOPNOTSUPP;
3310+
3311+ group = iommu_group_get(dev);
3312+ if (!group)
3313+ return -ENODEV;
3314+
3315+ mutex_lock(&group->mutex);
3316+ curr = xa_cmpxchg(&group->pasid_array, pasid, NULL, domain, GFP_KERNEL);
3317+ if (curr) {
3318+ ret = xa_err(curr) ? : -EBUSY;
3319+ goto out_unlock;
3320+ }
3321+
3322+ ret = __iommu_set_group_pasid(domain, group, pasid);
3323+ if (ret) {
3324+ __iommu_remove_group_pasid(group, pasid);
3325+ xa_erase(&group->pasid_array, pasid);
3326+ }
3327+out_unlock:
3328+ mutex_unlock(&group->mutex);
3329+ iommu_group_put(group);
3330+
3331+ return ret;
3332+}
3333+EXPORT_SYMBOL_GPL(iommu_attach_device_pasid);
3334+
3335+/*
3336+ * iommu_detach_device_pasid() - Detach the domain from pasid of device
3337+ * @domain: the iommu domain.
3338+ * @dev: the attached device.
3339+ * @pasid: the pasid of the device.
3340+ *
3341+ * The @domain must have been attached to @pasid of the @dev with
3342+ * iommu_attach_device_pasid().
3343+ */
3344+void iommu_detach_device_pasid(struct iommu_domain *domain, struct device *dev,
3345+ ioasid_t pasid)
3346+{
3347+ struct iommu_group *group = iommu_group_get(dev);
3348+
3349+ mutex_lock(&group->mutex);
3350+ __iommu_remove_group_pasid(group, pasid);
3351+ WARN_ON(xa_erase(&group->pasid_array, pasid) != domain);
3352+ mutex_unlock(&group->mutex);
3353+
3354+ iommu_group_put(group);
3355+}
3356+EXPORT_SYMBOL_GPL(iommu_detach_device_pasid);
3357+
3358+/*
3359+ * iommu_get_domain_for_dev_pasid() - Retrieve domain for @pasid of @dev
3360+ * @dev: the queried device
3361+ * @pasid: the pasid of the device
3362+ * @type: matched domain type, 0 for any match
3363+ *
3364+ * This is a variant of iommu_get_domain_for_dev(). It returns the existing
3365+ * domain attached to pasid of a device. Callers must hold a lock around this
3366+ * function, and both iommu_attach/detach_dev_pasid() whenever a domain of
3367+ * type is being manipulated. This API does not internally resolve races with
3368+ * attach/detach.
3369+ *
3370+ * Return: attached domain on success, NULL otherwise.
3371+ */
3372+struct iommu_domain *iommu_get_domain_for_dev_pasid(struct device *dev,
3373+ ioasid_t pasid,
3374+ unsigned int type)
3375+{
3376+ struct iommu_domain *domain;
3377+ struct iommu_group *group;
3378+
3379+ group = iommu_group_get(dev);
3380+ if (!group)
3381+ return NULL;
3382+
3383+ xa_lock(&group->pasid_array);
3384+ domain = xa_load(&group->pasid_array, pasid);
3385+ if (type && domain && domain->type != type)
3386+ domain = ERR_PTR(-EBUSY);
3387+ xa_unlock(&group->pasid_array);
3388+ iommu_group_put(group);
3389+
3390+ return domain;
3391+}
3392+EXPORT_SYMBOL_GPL(iommu_get_domain_for_dev_pasid);
3393+
3394+struct iommu_domain *iommu_sva_domain_alloc(struct device *dev,
3395+ struct mm_struct *mm)
3396+{
3397+ const struct iommu_ops *ops = dev_iommu_ops(dev);
3398+ struct iommu_domain *domain;
3399+
3400+ domain = ops->domain_alloc(IOMMU_DOMAIN_SVA);
3401+ if (!domain)
3402+ return NULL;
3403+
3404+ domain->type = IOMMU_DOMAIN_SVA;
3405+ mmgrab(mm);
3406+ domain->mm = mm;
3407+ domain->iopf_handler = iommu_sva_handle_iopf;
3408+ domain->fault_data = mm;
3409+
3410+ return domain;
3411+}
drivers/iommu/iommufd/Kconfig+44−0
@@ -0,0 +1,44 @@
1+# SPDX-License-Identifier: GPL-2.0-only
2+config IOMMUFD
3+ tristate "IOMMU Userspace API"
4+ select INTERVAL_TREE
5+ select INTERVAL_TREE_SPAN_ITER
6+ select IOMMU_API
7+ default n
8+ help
9+ Provides /dev/iommu, the user API to control the IOMMU subsystem as
10+ it relates to managing IO page tables that point at user space memory.
11+
12+ If you don't know what to do here, say N.
13+
14+if IOMMUFD
15+config IOMMUFD_VFIO_CONTAINER
16+ bool "IOMMUFD provides the VFIO container /dev/vfio/vfio"
17+ depends on VFIO && !VFIO_CONTAINER
18+ default VFIO && !VFIO_CONTAINER
19+ help
20+ IOMMUFD will provide /dev/vfio/vfio instead of VFIO. This relies on
21+ IOMMUFD providing compatibility emulation to give the same ioctls.
22+ It provides an option to build a kernel with legacy VFIO components
23+ removed.
24+
25+ IOMMUFD VFIO container emulation is known to lack certain features
26+ of the native VFIO container, such as no-IOMMU support, peer-to-peer
27+ DMA mapping, PPC IOMMU support, as well as other potentially
28+ undiscovered gaps. This option is currently intended for the
29+ purpose of testing IOMMUFD with unmodified userspace supporting VFIO
30+ and making use of the Type1 VFIO IOMMU backend. General purpose
31+ enabling of this option is currently discouraged.
32+
33+ Unless testing IOMMUFD, say N here.
34+
35+config IOMMUFD_TEST
36+ bool "IOMMU Userspace API Test support"
37+ depends on DEBUG_KERNEL
38+ depends on FAULT_INJECTION
39+ depends on RUNTIME_TESTING_MENU
40+ default n
41+ help
42+ This is dangerous, do not enable unless running
43+ tools/testing/selftests/iommu
44+endif
drivers/iommu/iommufd/Makefile+13−0
@@ -0,0 +1,13 @@
1+# SPDX-License-Identifier: GPL-2.0-only
2+iommufd-y := \
3+ device.o \
4+ hw_pagetable.o \
5+ io_pagetable.o \
6+ ioas.o \
7+ main.o \
8+ pages.o \
9+ vfio_compat.o
10+
11+iommufd-$(CONFIG_IOMMUFD_TEST) += selftest.o
12+
13+obj-$(CONFIG_IOMMUFD) += iommufd.o
drivers/iommu/iommufd/device.c+772−0
@@ -0,0 +1,772 @@
1+// SPDX-License-Identifier: GPL-2.0-only
2+/* Copyright (c) 2021-2022, NVIDIA CORPORATION & AFFILIATES
3+ */
4+#include <linux/iommufd.h>
5+#include <linux/slab.h>
6+#include <linux/iommu.h>
7+#include <linux/irqdomain.h>
8+
9+#include "io_pagetable.h"
10+#include "iommufd_private.h"
11+
12+static bool allow_unsafe_interrupts;
13+module_param(allow_unsafe_interrupts, bool, S_IRUGO | S_IWUSR);
14+MODULE_PARM_DESC(
15+ allow_unsafe_interrupts,
16+ "Allow IOMMUFD to bind to devices even if the platform cannot isolate "
17+ "the MSI interrupt window. Enabling this is a security weakness.");
18+
19+/*
20+ * A iommufd_device object represents the binding relationship between a
21+ * consuming driver and the iommufd. These objects are created/destroyed by
22+ * external drivers, not by userspace.
23+ */
24+struct iommufd_device {
25+ struct iommufd_object obj;
26+ struct iommufd_ctx *ictx;
27+ struct iommufd_hw_pagetable *hwpt;
28+ /* Head at iommufd_hw_pagetable::devices */
29+ struct list_head devices_item;
30+ /* always the physical device */
31+ struct device *dev;
32+ struct iommu_group *group;
33+ bool enforce_cache_coherency;
34+};
35+
36+void iommufd_device_destroy(struct iommufd_object *obj)
37+{
38+ struct iommufd_device *idev =
39+ container_of(obj, struct iommufd_device, obj);
40+
41+ iommu_device_release_dma_owner(idev->dev);
42+ iommu_group_put(idev->group);
43+ iommufd_ctx_put(idev->ictx);
44+}
45+
46+/**
47+ * iommufd_device_bind - Bind a physical device to an iommu fd
48+ * @ictx: iommufd file descriptor
49+ * @dev: Pointer to a physical device struct
50+ * @id: Output ID number to return to userspace for this device
51+ *
52+ * A successful bind establishes an ownership over the device and returns
53+ * struct iommufd_device pointer, otherwise returns error pointer.
54+ *
55+ * A driver using this API must set driver_managed_dma and must not touch
56+ * the device until this routine succeeds and establishes ownership.
57+ *
58+ * Binding a PCI device places the entire RID under iommufd control.
59+ *
60+ * The caller must undo this with iommufd_device_unbind()
61+ */
62+struct iommufd_device *iommufd_device_bind(struct iommufd_ctx *ictx,
63+ struct device *dev, u32 *id)
64+{
65+ struct iommufd_device *idev;
66+ struct iommu_group *group;
67+ int rc;
68+
69+ /*
70+ * iommufd always sets IOMMU_CACHE because we offer no way for userspace
71+ * to restore cache coherency.
72+ */
73+ if (!device_iommu_capable(dev, IOMMU_CAP_CACHE_COHERENCY))
74+ return ERR_PTR(-EINVAL);
75+
76+ group = iommu_group_get(dev);
77+ if (!group)
78+ return ERR_PTR(-ENODEV);
79+
80+ rc = iommu_device_claim_dma_owner(dev, ictx);
81+ if (rc)
82+ goto out_group_put;
83+
84+ idev = iommufd_object_alloc(ictx, idev, IOMMUFD_OBJ_DEVICE);
85+ if (IS_ERR(idev)) {
86+ rc = PTR_ERR(idev);
87+ goto out_release_owner;
88+ }
89+ idev->ictx = ictx;
90+ iommufd_ctx_get(ictx);
91+ idev->dev = dev;
92+ idev->enforce_cache_coherency =
93+ device_iommu_capable(dev, IOMMU_CAP_ENFORCE_CACHE_COHERENCY);
94+ /* The calling driver is a user until iommufd_device_unbind() */
95+ refcount_inc(&idev->obj.users);
96+ /* group refcount moves into iommufd_device */
97+ idev->group = group;
98+
99+ /*
100+ * If the caller fails after this success it must call
101+ * iommufd_unbind_device() which is safe since we hold this refcount.
102+ * This also means the device is a leaf in the graph and no other object
103+ * can take a reference on it.
104+ */
105+ iommufd_object_finalize(ictx, &idev->obj);
106+ *id = idev->obj.id;
107+ return idev;
108+
109+out_release_owner:
110+ iommu_device_release_dma_owner(dev);
111+out_group_put:
112+ iommu_group_put(group);
113+ return ERR_PTR(rc);
114+}
115+EXPORT_SYMBOL_NS_GPL(iommufd_device_bind, IOMMUFD);
116+
117+/**
118+ * iommufd_device_unbind - Undo iommufd_device_bind()
119+ * @idev: Device returned by iommufd_device_bind()
120+ *
121+ * Release the device from iommufd control. The DMA ownership will return back
122+ * to unowned with DMA controlled by the DMA API. This invalidates the
123+ * iommufd_device pointer, other APIs that consume it must not be called
124+ * concurrently.
125+ */
126+void iommufd_device_unbind(struct iommufd_device *idev)
127+{
128+ bool was_destroyed;
129+
130+ was_destroyed = iommufd_object_destroy_user(idev->ictx, &idev->obj);
131+ WARN_ON(!was_destroyed);
132+}
133+EXPORT_SYMBOL_NS_GPL(iommufd_device_unbind, IOMMUFD);
134+
135+static int iommufd_device_setup_msi(struct iommufd_device *idev,
136+ struct iommufd_hw_pagetable *hwpt,
137+ phys_addr_t sw_msi_start)
138+{
139+ int rc;
140+
141+ /*
142+ * If the IOMMU driver gives a IOMMU_RESV_SW_MSI then it is asking us to
143+ * call iommu_get_msi_cookie() on its behalf. This is necessary to setup
144+ * the MSI window so iommu_dma_prepare_msi() can install pages into our
145+ * domain after request_irq(). If it is not done interrupts will not
146+ * work on this domain.
147+ *
148+ * FIXME: This is conceptually broken for iommufd since we want to allow
149+ * userspace to change the domains, eg switch from an identity IOAS to a
150+ * DMA IOAS. There is currently no way to create a MSI window that
151+ * matches what the IRQ layer actually expects in a newly created
152+ * domain.
153+ */
154+ if (sw_msi_start != PHYS_ADDR_MAX && !hwpt->msi_cookie) {
155+ rc = iommu_get_msi_cookie(hwpt->domain, sw_msi_start);
156+ if (rc)
157+ return rc;
158+
159+ /*
160+ * iommu_get_msi_cookie() can only be called once per domain,
161+ * it returns -EBUSY on later calls.
162+ */
163+ hwpt->msi_cookie = true;
164+ }
165+
166+ /*
167+ * For historical compat with VFIO the insecure interrupt path is
168+ * allowed if the module parameter is set. Insecure means that a MemWr
169+ * operation from the device (eg a simple DMA) cannot trigger an
170+ * interrupt outside this iommufd context.
171+ */
172+ if (!device_iommu_capable(idev->dev, IOMMU_CAP_INTR_REMAP) &&
173+ !irq_domain_check_msi_remap()) {
174+ if (!allow_unsafe_interrupts)
175+ return -EPERM;
176+
177+ dev_warn(
178+ idev->dev,
179+ "MSI interrupts are not secure, they cannot be isolated by the platform. "
180+ "Check that platform features like interrupt remapping are enabled. "
181+ "Use the \"allow_unsafe_interrupts\" module parameter to override\n");
182+ }
183+ return 0;
184+}
185+
186+static bool iommufd_hw_pagetable_has_group(struct iommufd_hw_pagetable *hwpt,
187+ struct iommu_group *group)
188+{
189+ struct iommufd_device *cur_dev;
190+
191+ list_for_each_entry(cur_dev, &hwpt->devices, devices_item)
192+ if (cur_dev->group == group)
193+ return true;
194+ return false;
195+}
196+
197+static int iommufd_device_do_attach(struct iommufd_device *idev,
198+ struct iommufd_hw_pagetable *hwpt)
199+{
200+ phys_addr_t sw_msi_start = PHYS_ADDR_MAX;
201+ int rc;
202+
203+ mutex_lock(&hwpt->devices_lock);
204+
205+ /*
206+ * Try to upgrade the domain we have, it is an iommu driver bug to
207+ * report IOMMU_CAP_ENFORCE_CACHE_COHERENCY but fail
208+ * enforce_cache_coherency when there are no devices attached to the
209+ * domain.
210+ */
211+ if (idev->enforce_cache_coherency && !hwpt->enforce_cache_coherency) {
212+ if (hwpt->domain->ops->enforce_cache_coherency)
213+ hwpt->enforce_cache_coherency =
214+ hwpt->domain->ops->enforce_cache_coherency(
215+ hwpt->domain);
216+ if (!hwpt->enforce_cache_coherency) {
217+ WARN_ON(list_empty(&hwpt->devices));
218+ rc = -EINVAL;
219+ goto out_unlock;
220+ }
221+ }
222+
223+ rc = iopt_table_enforce_group_resv_regions(&hwpt->ioas->iopt, idev->dev,
224+ idev->group, &sw_msi_start);
225+ if (rc)
226+ goto out_unlock;
227+
228+ rc = iommufd_device_setup_msi(idev, hwpt, sw_msi_start);
229+ if (rc)
230+ goto out_iova;
231+
232+ /*
233+ * FIXME: Hack around missing a device-centric iommu api, only attach to
234+ * the group once for the first device that is in the group.
235+ */
236+ if (!iommufd_hw_pagetable_has_group(hwpt, idev->group)) {
237+ rc = iommu_attach_group(hwpt->domain, idev->group);
238+ if (rc)
239+ goto out_iova;
240+
241+ if (list_empty(&hwpt->devices)) {
242+ rc = iopt_table_add_domain(&hwpt->ioas->iopt,
243+ hwpt->domain);
244+ if (rc)
245+ goto out_detach;
246+ }
247+ }
248+
249+ idev->hwpt = hwpt;
250+ refcount_inc(&hwpt->obj.users);
251+ list_add(&idev->devices_item, &hwpt->devices);
252+ mutex_unlock(&hwpt->devices_lock);
253+ return 0;
254+
255+out_detach:
256+ iommu_detach_group(hwpt->domain, idev->group);
257+out_iova:
258+ iopt_remove_reserved_iova(&hwpt->ioas->iopt, idev->dev);
259+out_unlock:
260+ mutex_unlock(&hwpt->devices_lock);
261+ return rc;
262+}
263+
264+/*
265+ * When automatically managing the domains we search for a compatible domain in
266+ * the iopt and if one is found use it, otherwise create a new domain.
267+ * Automatic domain selection will never pick a manually created domain.
268+ */
269+static int iommufd_device_auto_get_domain(struct iommufd_device *idev,
270+ struct iommufd_ioas *ioas)
271+{
272+ struct iommufd_hw_pagetable *hwpt;
273+ int rc;
274+
275+ /*
276+ * There is no differentiation when domains are allocated, so any domain
277+ * that is willing to attach to the device is interchangeable with any
278+ * other.
279+ */
280+ mutex_lock(&ioas->mutex);
281+ list_for_each_entry(hwpt, &ioas->hwpt_list, hwpt_item) {
282+ if (!hwpt->auto_domain)
283+ continue;
284+
285+ rc = iommufd_device_do_attach(idev, hwpt);
286+
287+ /*
288+ * -EINVAL means the domain is incompatible with the device.
289+ * Other error codes should propagate to userspace as failure.
290+ * Success means the domain is attached.
291+ */
292+ if (rc == -EINVAL)
293+ continue;
294+ goto out_unlock;
295+ }
296+
297+ hwpt = iommufd_hw_pagetable_alloc(idev->ictx, ioas, idev->dev);
298+ if (IS_ERR(hwpt)) {
299+ rc = PTR_ERR(hwpt);
300+ goto out_unlock;
301+ }
302+ hwpt->auto_domain = true;
303+
304+ rc = iommufd_device_do_attach(idev, hwpt);
305+ if (rc)
306+ goto out_abort;
307+ list_add_tail(&hwpt->hwpt_item, &ioas->hwpt_list);
308+
309+ mutex_unlock(&ioas->mutex);
310+ iommufd_object_finalize(idev->ictx, &hwpt->obj);
311+ return 0;
312+
313+out_abort:
314+ iommufd_object_abort_and_destroy(idev->ictx, &hwpt->obj);
315+out_unlock:
316+ mutex_unlock(&ioas->mutex);
317+ return rc;
318+}
319+
320+/**
321+ * iommufd_device_attach - Connect a device from an iommu_domain
322+ * @idev: device to attach
323+ * @pt_id: Input a IOMMUFD_OBJ_IOAS, or IOMMUFD_OBJ_HW_PAGETABLE
324+ * Output the IOMMUFD_OBJ_HW_PAGETABLE ID
325+ *
326+ * This connects the device to an iommu_domain, either automatically or manually
327+ * selected. Once this completes the device could do DMA.
328+ *
329+ * The caller should return the resulting pt_id back to userspace.
330+ * This function is undone by calling iommufd_device_detach().
331+ */
332+int iommufd_device_attach(struct iommufd_device *idev, u32 *pt_id)
333+{
334+ struct iommufd_object *pt_obj;
335+ int rc;
336+
337+ pt_obj = iommufd_get_object(idev->ictx, *pt_id, IOMMUFD_OBJ_ANY);
338+ if (IS_ERR(pt_obj))
339+ return PTR_ERR(pt_obj);
340+
341+ switch (pt_obj->type) {
342+ case IOMMUFD_OBJ_HW_PAGETABLE: {
343+ struct iommufd_hw_pagetable *hwpt =
344+ container_of(pt_obj, struct iommufd_hw_pagetable, obj);
345+
346+ rc = iommufd_device_do_attach(idev, hwpt);
347+ if (rc)
348+ goto out_put_pt_obj;
349+
350+ mutex_lock(&hwpt->ioas->mutex);
351+ list_add_tail(&hwpt->hwpt_item, &hwpt->ioas->hwpt_list);
352+ mutex_unlock(&hwpt->ioas->mutex);
353+ break;
354+ }
355+ case IOMMUFD_OBJ_IOAS: {
356+ struct iommufd_ioas *ioas =
357+ container_of(pt_obj, struct iommufd_ioas, obj);
358+
359+ rc = iommufd_device_auto_get_domain(idev, ioas);
360+ if (rc)
361+ goto out_put_pt_obj;
362+ break;
363+ }
364+ default:
365+ rc = -EINVAL;
366+ goto out_put_pt_obj;
367+ }
368+
369+ refcount_inc(&idev->obj.users);
370+ *pt_id = idev->hwpt->obj.id;
371+ rc = 0;
372+
373+out_put_pt_obj:
374+ iommufd_put_object(pt_obj);
375+ return rc;
376+}
377+EXPORT_SYMBOL_NS_GPL(iommufd_device_attach, IOMMUFD);
378+
379+/**
380+ * iommufd_device_detach - Disconnect a device to an iommu_domain
381+ * @idev: device to detach
382+ *
383+ * Undo iommufd_device_attach(). This disconnects the idev from the previously
384+ * attached pt_id. The device returns back to a blocked DMA translation.
385+ */
386+void iommufd_device_detach(struct iommufd_device *idev)
387+{
388+ struct iommufd_hw_pagetable *hwpt = idev->hwpt;
389+
390+ mutex_lock(&hwpt->ioas->mutex);
391+ mutex_lock(&hwpt->devices_lock);
392+ list_del(&idev->devices_item);
393+ if (!iommufd_hw_pagetable_has_group(hwpt, idev->group)) {
394+ if (list_empty(&hwpt->devices)) {
395+ iopt_table_remove_domain(&hwpt->ioas->iopt,
396+ hwpt->domain);
397+ list_del(&hwpt->hwpt_item);
398+ }
399+ iommu_detach_group(hwpt->domain, idev->group);
400+ }
401+ iopt_remove_reserved_iova(&hwpt->ioas->iopt, idev->dev);
402+ mutex_unlock(&hwpt->devices_lock);
403+ mutex_unlock(&hwpt->ioas->mutex);
404+
405+ if (hwpt->auto_domain)
406+ iommufd_object_destroy_user(idev->ictx, &hwpt->obj);
407+ else
408+ refcount_dec(&hwpt->obj.users);
409+
410+ idev->hwpt = NULL;
411+
412+ refcount_dec(&idev->obj.users);
413+}
414+EXPORT_SYMBOL_NS_GPL(iommufd_device_detach, IOMMUFD);
415+
416+void iommufd_access_destroy_object(struct iommufd_object *obj)
417+{
418+ struct iommufd_access *access =
419+ container_of(obj, struct iommufd_access, obj);
420+
421+ iopt_remove_access(&access->ioas->iopt, access);
422+ iommufd_ctx_put(access->ictx);
423+ refcount_dec(&access->ioas->obj.users);
424+}
425+
426+/**
427+ * iommufd_access_create - Create an iommufd_access
428+ * @ictx: iommufd file descriptor
429+ * @ioas_id: ID for a IOMMUFD_OBJ_IOAS
430+ * @ops: Driver's ops to associate with the access
431+ * @data: Opaque data to pass into ops functions
432+ *
433+ * An iommufd_access allows a driver to read/write to the IOAS without using
434+ * DMA. The underlying CPU memory can be accessed using the
435+ * iommufd_access_pin_pages() or iommufd_access_rw() functions.
436+ *
437+ * The provided ops are required to use iommufd_access_pin_pages().
438+ */
439+struct iommufd_access *
440+iommufd_access_create(struct iommufd_ctx *ictx, u32 ioas_id,
441+ const struct iommufd_access_ops *ops, void *data)
442+{
443+ struct iommufd_access *access;
444+ struct iommufd_object *obj;
445+ int rc;
446+
447+ /*
448+ * There is no uAPI for the access object, but to keep things symmetric
449+ * use the object infrastructure anyhow.
450+ */
451+ access = iommufd_object_alloc(ictx, access, IOMMUFD_OBJ_ACCESS);
452+ if (IS_ERR(access))
453+ return access;
454+
455+ access->data = data;
456+ access->ops = ops;
457+
458+ obj = iommufd_get_object(ictx, ioas_id, IOMMUFD_OBJ_IOAS);
459+ if (IS_ERR(obj)) {
460+ rc = PTR_ERR(obj);
461+ goto out_abort;
462+ }
463+ access->ioas = container_of(obj, struct iommufd_ioas, obj);
464+ iommufd_ref_to_users(obj);
465+
466+ if (ops->needs_pin_pages)
467+ access->iova_alignment = PAGE_SIZE;
468+ else
469+ access->iova_alignment = 1;
470+ rc = iopt_add_access(&access->ioas->iopt, access);
471+ if (rc)
472+ goto out_put_ioas;
473+
474+ /* The calling driver is a user until iommufd_access_destroy() */
475+ refcount_inc(&access->obj.users);
476+ access->ictx = ictx;
477+ iommufd_ctx_get(ictx);
478+ iommufd_object_finalize(ictx, &access->obj);
479+ return access;
480+out_put_ioas:
481+ refcount_dec(&access->ioas->obj.users);
482+out_abort:
483+ iommufd_object_abort(ictx, &access->obj);
484+ return ERR_PTR(rc);
485+}
486+EXPORT_SYMBOL_NS_GPL(iommufd_access_create, IOMMUFD);
487+
488+/**
489+ * iommufd_access_destroy - Destroy an iommufd_access
490+ * @access: The access to destroy
491+ *
492+ * The caller must stop using the access before destroying it.
493+ */
494+void iommufd_access_destroy(struct iommufd_access *access)
495+{
496+ bool was_destroyed;
497+
498+ was_destroyed = iommufd_object_destroy_user(access->ictx, &access->obj);
499+ WARN_ON(!was_destroyed);
500+}
501+EXPORT_SYMBOL_NS_GPL(iommufd_access_destroy, IOMMUFD);
502+
503+/**
504+ * iommufd_access_notify_unmap - Notify users of an iopt to stop using it
505+ * @iopt: iopt to work on
506+ * @iova: Starting iova in the iopt
507+ * @length: Number of bytes
508+ *
509+ * After this function returns there should be no users attached to the pages
510+ * linked to this iopt that intersect with iova,length. Anyone that has attached
511+ * a user through iopt_access_pages() needs to detach it through
512+ * iommufd_access_unpin_pages() before this function returns.
513+ *
514+ * iommufd_access_destroy() will wait for any outstanding unmap callback to
515+ * complete. Once iommufd_access_destroy() no unmap ops are running or will
516+ * run in the future. Due to this a driver must not create locking that prevents
517+ * unmap to complete while iommufd_access_destroy() is running.
518+ */
519+void iommufd_access_notify_unmap(struct io_pagetable *iopt, unsigned long iova,
520+ unsigned long length)
521+{
522+ struct iommufd_ioas *ioas =
523+ container_of(iopt, struct iommufd_ioas, iopt);
524+ struct iommufd_access *access;
525+ unsigned long index;
526+
527+ xa_lock(&ioas->iopt.access_list);
528+ xa_for_each(&ioas->iopt.access_list, index, access) {
529+ if (!iommufd_lock_obj(&access->obj))
530+ continue;
531+ xa_unlock(&ioas->iopt.access_list);
532+
533+ access->ops->unmap(access->data, iova, length);
534+
535+ iommufd_put_object(&access->obj);
536+ xa_lock(&ioas->iopt.access_list);
537+ }
538+ xa_unlock(&ioas->iopt.access_list);
539+}
540+
541+/**
542+ * iommufd_access_unpin_pages() - Undo iommufd_access_pin_pages
543+ * @access: IOAS access to act on
544+ * @iova: Starting IOVA
545+ * @length: Number of bytes to access
546+ *
547+ * Return the struct page's. The caller must stop accessing them before calling
548+ * this. The iova/length must exactly match the one provided to access_pages.
549+ */
550+void iommufd_access_unpin_pages(struct iommufd_access *access,
551+ unsigned long iova, unsigned long length)
552+{
553+ struct io_pagetable *iopt = &access->ioas->iopt;
554+ struct iopt_area_contig_iter iter;
555+ unsigned long last_iova;
556+ struct iopt_area *area;
557+
558+ if (WARN_ON(!length) ||
559+ WARN_ON(check_add_overflow(iova, length - 1, &last_iova)))
560+ return;
561+
562+ down_read(&iopt->iova_rwsem);
563+ iopt_for_each_contig_area(&iter, area, iopt, iova, last_iova)
564+ iopt_area_remove_access(
565+ area, iopt_area_iova_to_index(area, iter.cur_iova),
566+ iopt_area_iova_to_index(
567+ area,
568+ min(last_iova, iopt_area_last_iova(area))));
569+ up_read(&iopt->iova_rwsem);
570+ WARN_ON(!iopt_area_contig_done(&iter));
571+}
572+EXPORT_SYMBOL_NS_GPL(iommufd_access_unpin_pages, IOMMUFD);
573+
574+static bool iopt_area_contig_is_aligned(struct iopt_area_contig_iter *iter)
575+{
576+ if (iopt_area_start_byte(iter->area, iter->cur_iova) % PAGE_SIZE)
577+ return false;
578+
579+ if (!iopt_area_contig_done(iter) &&
580+ (iopt_area_start_byte(iter->area, iopt_area_last_iova(iter->area)) %
581+ PAGE_SIZE) != (PAGE_SIZE - 1))
582+ return false;
583+ return true;
584+}
585+
586+static bool check_area_prot(struct iopt_area *area, unsigned int flags)
587+{
588+ if (flags & IOMMUFD_ACCESS_RW_WRITE)
589+ return area->iommu_prot & IOMMU_WRITE;
590+ return area->iommu_prot & IOMMU_READ;
591+}
592+
593+/**
594+ * iommufd_access_pin_pages() - Return a list of pages under the iova
595+ * @access: IOAS access to act on
596+ * @iova: Starting IOVA
597+ * @length: Number of bytes to access
598+ * @out_pages: Output page list
599+ * @flags: IOPMMUFD_ACCESS_RW_* flags
600+ *
601+ * Reads @length bytes starting at iova and returns the struct page * pointers.
602+ * These can be kmap'd by the caller for CPU access.
603+ *
604+ * The caller must perform iommufd_access_unpin_pages() when done to balance
605+ * this.
606+ *
607+ * This API always requires a page aligned iova. This happens naturally if the
608+ * ioas alignment is >= PAGE_SIZE and the iova is PAGE_SIZE aligned. However
609+ * smaller alignments have corner cases where this API can fail on otherwise
610+ * aligned iova.
611+ */
612+int iommufd_access_pin_pages(struct iommufd_access *access, unsigned long iova,
613+ unsigned long length, struct page **out_pages,
614+ unsigned int flags)
615+{
616+ struct io_pagetable *iopt = &access->ioas->iopt;
617+ struct iopt_area_contig_iter iter;
618+ unsigned long last_iova;
619+ struct iopt_area *area;
620+ int rc;
621+
622+ /* Driver's ops don't support pin_pages */
623+ if (IS_ENABLED(CONFIG_IOMMUFD_TEST) &&
624+ WARN_ON(access->iova_alignment != PAGE_SIZE || !access->ops->unmap))
625+ return -EINVAL;
626+
627+ if (!length)
628+ return -EINVAL;
629+ if (check_add_overflow(iova, length - 1, &last_iova))
630+ return -EOVERFLOW;
631+
632+ down_read(&iopt->iova_rwsem);
633+ iopt_for_each_contig_area(&iter, area, iopt, iova, last_iova) {
634+ unsigned long last = min(last_iova, iopt_area_last_iova(area));
635+ unsigned long last_index = iopt_area_iova_to_index(area, last);
636+ unsigned long index =
637+ iopt_area_iova_to_index(area, iter.cur_iova);
638+
639+ if (area->prevent_access ||
640+ !iopt_area_contig_is_aligned(&iter)) {
641+ rc = -EINVAL;
642+ goto err_remove;
643+ }
644+
645+ if (!check_area_prot(area, flags)) {
646+ rc = -EPERM;
647+ goto err_remove;
648+ }
649+
650+ rc = iopt_area_add_access(area, index, last_index, out_pages,
651+ flags);
652+ if (rc)
653+ goto err_remove;
654+ out_pages += last_index - index + 1;
655+ }
656+ if (!iopt_area_contig_done(&iter)) {
657+ rc = -ENOENT;
658+ goto err_remove;
659+ }
660+
661+ up_read(&iopt->iova_rwsem);
662+ return 0;
663+
664+err_remove:
665+ if (iova < iter.cur_iova) {
666+ last_iova = iter.cur_iova - 1;
667+ iopt_for_each_contig_area(&iter, area, iopt, iova, last_iova)
668+ iopt_area_remove_access(
669+ area,
670+ iopt_area_iova_to_index(area, iter.cur_iova),
671+ iopt_area_iova_to_index(
672+ area, min(last_iova,
673+ iopt_area_last_iova(area))));
674+ }
675+ up_read(&iopt->iova_rwsem);
676+ return rc;
677+}
678+EXPORT_SYMBOL_NS_GPL(iommufd_access_pin_pages, IOMMUFD);
679+
680+/**
681+ * iommufd_access_rw - Read or write data under the iova
682+ * @access: IOAS access to act on
683+ * @iova: Starting IOVA
684+ * @data: Kernel buffer to copy to/from
685+ * @length: Number of bytes to access
686+ * @flags: IOMMUFD_ACCESS_RW_* flags
687+ *
688+ * Copy kernel to/from data into the range given by IOVA/length. If flags
689+ * indicates IOMMUFD_ACCESS_RW_KTHREAD then a large copy can be optimized
690+ * by changing it into copy_to/from_user().
691+ */
692+int iommufd_access_rw(struct iommufd_access *access, unsigned long iova,
693+ void *data, size_t length, unsigned int flags)
694+{
695+ struct io_pagetable *iopt = &access->ioas->iopt;
696+ struct iopt_area_contig_iter iter;
697+ struct iopt_area *area;
698+ unsigned long last_iova;
699+ int rc;
700+
701+ if (!length)
702+ return -EINVAL;
703+ if (check_add_overflow(iova, length - 1, &last_iova))
704+ return -EOVERFLOW;
705+
706+ down_read(&iopt->iova_rwsem);
707+ iopt_for_each_contig_area(&iter, area, iopt, iova, last_iova) {
708+ unsigned long last = min(last_iova, iopt_area_last_iova(area));
709+ unsigned long bytes = (last - iter.cur_iova) + 1;
710+
711+ if (area->prevent_access) {
712+ rc = -EINVAL;
713+ goto err_out;
714+ }
715+
716+ if (!check_area_prot(area, flags)) {
717+ rc = -EPERM;
718+ goto err_out;
719+ }
720+
721+ rc = iopt_pages_rw_access(
722+ area->pages, iopt_area_start_byte(area, iter.cur_iova),
723+ data, bytes, flags);
724+ if (rc)
725+ goto err_out;
726+ data += bytes;
727+ }
728+ if (!iopt_area_contig_done(&iter))
729+ rc = -ENOENT;
730+err_out:
731+ up_read(&iopt->iova_rwsem);
732+ return rc;
733+}
734+EXPORT_SYMBOL_NS_GPL(iommufd_access_rw, IOMMUFD);
735+
736+#ifdef CONFIG_IOMMUFD_TEST
737+/*
738+ * Creating a real iommufd_device is too hard, bypass creating a iommufd_device
739+ * and go directly to attaching a domain.
740+ */
741+struct iommufd_hw_pagetable *
742+iommufd_device_selftest_attach(struct iommufd_ctx *ictx,
743+ struct iommufd_ioas *ioas,
744+ struct device *mock_dev)
745+{
746+ struct iommufd_hw_pagetable *hwpt;
747+ int rc;
748+
749+ hwpt = iommufd_hw_pagetable_alloc(ictx, ioas, mock_dev);
750+ if (IS_ERR(hwpt))
751+ return hwpt;
752+
753+ rc = iopt_table_add_domain(&hwpt->ioas->iopt, hwpt->domain);
754+ if (rc)
755+ goto out_hwpt;
756+
757+ refcount_inc(&hwpt->obj.users);
758+ iommufd_object_finalize(ictx, &hwpt->obj);
759+ return hwpt;
760+
761+out_hwpt:
762+ iommufd_object_abort_and_destroy(ictx, &hwpt->obj);
763+ return ERR_PTR(rc);
764+}
765+
766+void iommufd_device_selftest_detach(struct iommufd_ctx *ictx,
767+ struct iommufd_hw_pagetable *hwpt)
768+{
769+ iopt_table_remove_domain(&hwpt->ioas->iopt, hwpt->domain);
770+ refcount_dec(&hwpt->obj.users);
771+}
772+#endif
drivers/iommu/iommufd/double_span.h+53−0
@@ -0,0 +1,53 @@
1+/* SPDX-License-Identifier: GPL-2.0 */
2+/* Copyright (c) 2022, NVIDIA CORPORATION & AFFILIATES.
3+ */
4+#ifndef __IOMMUFD_DOUBLE_SPAN_H
5+#define __IOMMUFD_DOUBLE_SPAN_H
6+
7+#include <linux/interval_tree.h>
8+
9+/*
10+ * This is a variation of the general interval_tree_span_iter that computes the
11+ * spans over the union of two different interval trees. Used ranges are broken
12+ * up and reported based on the tree that provides the interval. The first span
13+ * always takes priority. Like interval_tree_span_iter it is greedy and the same
14+ * value of is_used will not repeat on two iteration cycles.
15+ */
16+struct interval_tree_double_span_iter {
17+ struct rb_root_cached *itrees[2];
18+ struct interval_tree_span_iter spans[2];
19+ union {
20+ unsigned long start_hole;
21+ unsigned long start_used;
22+ };
23+ union {
24+ unsigned long last_hole;
25+ unsigned long last_used;
26+ };
27+ /* 0 = hole, 1 = used span[0], 2 = used span[1], -1 done iteration */
28+ int is_used;
29+};
30+
31+void interval_tree_double_span_iter_update(
32+ struct interval_tree_double_span_iter *iter);
33+void interval_tree_double_span_iter_first(
34+ struct interval_tree_double_span_iter *iter,
35+ struct rb_root_cached *itree1, struct rb_root_cached *itree2,
36+ unsigned long first_index, unsigned long last_index);
37+void interval_tree_double_span_iter_next(
38+ struct interval_tree_double_span_iter *iter);
39+
40+static inline bool
41+interval_tree_double_span_iter_done(struct interval_tree_double_span_iter *state)
42+{
43+ return state->is_used == -1;
44+}
45+
46+#define interval_tree_for_each_double_span(span, itree1, itree2, first_index, \
47+ last_index) \
48+ for (interval_tree_double_span_iter_first(span, itree1, itree2, \
49+ first_index, last_index); \
50+ !interval_tree_double_span_iter_done(span); \
51+ interval_tree_double_span_iter_next(span))
52+
53+#endif
drivers/iommu/iommufd/hw_pagetable.c+57−0
@@ -0,0 +1,57 @@
1+// SPDX-License-Identifier: GPL-2.0-only
2+/*
3+ * Copyright (c) 2021-2022, NVIDIA CORPORATION & AFFILIATES
4+ */
5+#include <linux/iommu.h>
6+
7+#include "iommufd_private.h"
8+
9+void iommufd_hw_pagetable_destroy(struct iommufd_object *obj)
10+{
11+ struct iommufd_hw_pagetable *hwpt =
12+ container_of(obj, struct iommufd_hw_pagetable, obj);
13+
14+ WARN_ON(!list_empty(&hwpt->devices));
15+
16+ iommu_domain_free(hwpt->domain);
17+ refcount_dec(&hwpt->ioas->obj.users);
18+ mutex_destroy(&hwpt->devices_lock);
19+}
20+
21+/**
22+ * iommufd_hw_pagetable_alloc() - Get an iommu_domain for a device
23+ * @ictx: iommufd context
24+ * @ioas: IOAS to associate the domain with
25+ * @dev: Device to get an iommu_domain for
26+ *
27+ * Allocate a new iommu_domain and return it as a hw_pagetable.
28+ */
29+struct iommufd_hw_pagetable *
30+iommufd_hw_pagetable_alloc(struct iommufd_ctx *ictx, struct iommufd_ioas *ioas,
31+ struct device *dev)
32+{
33+ struct iommufd_hw_pagetable *hwpt;
34+ int rc;
35+
36+ hwpt = iommufd_object_alloc(ictx, hwpt, IOMMUFD_OBJ_HW_PAGETABLE);
37+ if (IS_ERR(hwpt))
38+ return hwpt;
39+
40+ hwpt->domain = iommu_domain_alloc(dev->bus);
41+ if (!hwpt->domain) {
42+ rc = -ENOMEM;
43+ goto out_abort;
44+ }
45+
46+ INIT_LIST_HEAD(&hwpt->devices);
47+ INIT_LIST_HEAD(&hwpt->hwpt_item);
48+ mutex_init(&hwpt->devices_lock);
49+ /* Pairs with iommufd_hw_pagetable_destroy() */
50+ refcount_inc(&ioas->obj.users);
51+ hwpt->ioas = ioas;
52+ return hwpt;
53+
54+out_abort:
55+ iommufd_object_abort(ictx, &hwpt->obj);
56+ return ERR_PTR(rc);
57+}
drivers/iommu/iommufd/io_pagetable.c+1216−0
@@ -0,0 +1,1216 @@
1+// SPDX-License-Identifier: GPL-2.0
2+/* Copyright (c) 2021-2022, NVIDIA CORPORATION & AFFILIATES.
3+ *
4+ * The io_pagetable is the top of datastructure that maps IOVA's to PFNs. The
5+ * PFNs can be placed into an iommu_domain, or returned to the caller as a page
6+ * list for access by an in-kernel user.
7+ *
8+ * The datastructure uses the iopt_pages to optimize the storage of the PFNs
9+ * between the domains and xarray.
10+ */
11+#include <linux/iommufd.h>
12+#include <linux/lockdep.h>
13+#include <linux/iommu.h>
14+#include <linux/sched/mm.h>
15+#include <linux/err.h>
16+#include <linux/slab.h>
17+#include <linux/errno.h>
18+
19+#include "io_pagetable.h"
20+#include "double_span.h"
21+
22+struct iopt_pages_list {
23+ struct iopt_pages *pages;
24+ struct iopt_area *area;
25+ struct list_head next;
26+ unsigned long start_byte;
27+ unsigned long length;
28+};
29+
30+struct iopt_area *iopt_area_contig_init(struct iopt_area_contig_iter *iter,
31+ struct io_pagetable *iopt,
32+ unsigned long iova,
33+ unsigned long last_iova)
34+{
35+ lockdep_assert_held(&iopt->iova_rwsem);
36+
37+ iter->cur_iova = iova;
38+ iter->last_iova = last_iova;
39+ iter->area = iopt_area_iter_first(iopt, iova, iova);
40+ if (!iter->area)
41+ return NULL;
42+ if (!iter->area->pages) {
43+ iter->area = NULL;
44+ return NULL;
45+ }
46+ return iter->area;
47+}
48+
49+struct iopt_area *iopt_area_contig_next(struct iopt_area_contig_iter *iter)
50+{
51+ unsigned long last_iova;
52+
53+ if (!iter->area)
54+ return NULL;
55+ last_iova = iopt_area_last_iova(iter->area);
56+ if (iter->last_iova <= last_iova)
57+ return NULL;
58+
59+ iter->cur_iova = last_iova + 1;
60+ iter->area = iopt_area_iter_next(iter->area, iter->cur_iova,
61+ iter->last_iova);
62+ if (!iter->area)
63+ return NULL;
64+ if (iter->cur_iova != iopt_area_iova(iter->area) ||
65+ !iter->area->pages) {
66+ iter->area = NULL;
67+ return NULL;
68+ }
69+ return iter->area;
70+}
71+
72+static bool __alloc_iova_check_hole(struct interval_tree_double_span_iter *span,
73+ unsigned long length,
74+ unsigned long iova_alignment,
75+ unsigned long page_offset)
76+{
77+ if (span->is_used || span->last_hole - span->start_hole < length - 1)
78+ return false;
79+
80+ span->start_hole = ALIGN(span->start_hole, iova_alignment) |
81+ page_offset;
82+ if (span->start_hole > span->last_hole ||
83+ span->last_hole - span->start_hole < length - 1)
84+ return false;
85+ return true;
86+}
87+
88+static bool __alloc_iova_check_used(struct interval_tree_span_iter *span,
89+ unsigned long length,
90+ unsigned long iova_alignment,
91+ unsigned long page_offset)
92+{
93+ if (span->is_hole || span->last_used - span->start_used < length - 1)
94+ return false;
95+
96+ span->start_used = ALIGN(span->start_used, iova_alignment) |
97+ page_offset;
98+ if (span->start_used > span->last_used ||
99+ span->last_used - span->start_used < length - 1)
100+ return false;
101+ return true;
102+}
103+
104+/*
105+ * Automatically find a block of IOVA that is not being used and not reserved.
106+ * Does not return a 0 IOVA even if it is valid.
107+ */
108+static int iopt_alloc_iova(struct io_pagetable *iopt, unsigned long *iova,
109+ unsigned long uptr, unsigned long length)
110+{
111+ unsigned long page_offset = uptr % PAGE_SIZE;
112+ struct interval_tree_double_span_iter used_span;
113+ struct interval_tree_span_iter allowed_span;
114+ unsigned long iova_alignment;
115+
116+ lockdep_assert_held(&iopt->iova_rwsem);
117+
118+ /* Protect roundup_pow-of_two() from overflow */
119+ if (length == 0 || length >= ULONG_MAX / 2)
120+ return -EOVERFLOW;
121+
122+ /*
123+ * Keep alignment present in the uptr when building the IOVA, this
124+ * increases the chance we can map a THP.
125+ */
126+ if (!uptr)
127+ iova_alignment = roundup_pow_of_two(length);
128+ else
129+ iova_alignment = min_t(unsigned long,
130+ roundup_pow_of_two(length),
131+ 1UL << __ffs64(uptr));
132+
133+ if (iova_alignment < iopt->iova_alignment)
134+ return -EINVAL;
135+
136+ interval_tree_for_each_span(&allowed_span, &iopt->allowed_itree,
137+ PAGE_SIZE, ULONG_MAX - PAGE_SIZE) {
138+ if (RB_EMPTY_ROOT(&iopt->allowed_itree.rb_root)) {
139+ allowed_span.start_used = PAGE_SIZE;
140+ allowed_span.last_used = ULONG_MAX - PAGE_SIZE;
141+ allowed_span.is_hole = false;
142+ }
143+
144+ if (!__alloc_iova_check_used(&allowed_span, length,
145+ iova_alignment, page_offset))
146+ continue;
147+
148+ interval_tree_for_each_double_span(
149+ &used_span, &iopt->reserved_itree, &iopt->area_itree,
150+ allowed_span.start_used, allowed_span.last_used) {
151+ if (!__alloc_iova_check_hole(&used_span, length,
152+ iova_alignment,
153+ page_offset))
154+ continue;
155+
156+ *iova = used_span.start_hole;
157+ return 0;
158+ }
159+ }
160+ return -ENOSPC;
161+}
162+
163+static int iopt_check_iova(struct io_pagetable *iopt, unsigned long iova,
164+ unsigned long length)
165+{
166+ unsigned long last;
167+
168+ lockdep_assert_held(&iopt->iova_rwsem);
169+
170+ if ((iova & (iopt->iova_alignment - 1)))
171+ return -EINVAL;
172+
173+ if (check_add_overflow(iova, length - 1, &last))
174+ return -EOVERFLOW;
175+
176+ /* No reserved IOVA intersects the range */
177+ if (iopt_reserved_iter_first(iopt, iova, last))
178+ return -EINVAL;
179+
180+ /* Check that there is not already a mapping in the range */
181+ if (iopt_area_iter_first(iopt, iova, last))
182+ return -EEXIST;
183+ return 0;
184+}
185+
186+/*
187+ * The area takes a slice of the pages from start_bytes to start_byte + length
188+ */
189+static int iopt_insert_area(struct io_pagetable *iopt, struct iopt_area *area,
190+ struct iopt_pages *pages, unsigned long iova,
191+ unsigned long start_byte, unsigned long length,
192+ int iommu_prot)
193+{
194+ lockdep_assert_held_write(&iopt->iova_rwsem);
195+
196+ if ((iommu_prot & IOMMU_WRITE) && !pages->writable)
197+ return -EPERM;
198+
199+ area->iommu_prot = iommu_prot;
200+ area->page_offset = start_byte % PAGE_SIZE;
201+ if (area->page_offset & (iopt->iova_alignment - 1))
202+ return -EINVAL;
203+
204+ area->node.start = iova;
205+ if (check_add_overflow(iova, length - 1, &area->node.last))
206+ return -EOVERFLOW;
207+
208+ area->pages_node.start = start_byte / PAGE_SIZE;
209+ if (check_add_overflow(start_byte, length - 1, &area->pages_node.last))
210+ return -EOVERFLOW;
211+ area->pages_node.last = area->pages_node.last / PAGE_SIZE;
212+ if (WARN_ON(area->pages_node.last >= pages->npages))
213+ return -EOVERFLOW;
214+
215+ /*
216+ * The area is inserted with a NULL pages indicating it is not fully
217+ * initialized yet.
218+ */
219+ area->iopt = iopt;
220+ interval_tree_insert(&area->node, &iopt->area_itree);
221+ return 0;
222+}
223+
224+static int iopt_alloc_area_pages(struct io_pagetable *iopt,
225+ struct list_head *pages_list,
226+ unsigned long length, unsigned long *dst_iova,
227+ int iommu_prot, unsigned int flags)
228+{
229+ struct iopt_pages_list *elm;
230+ unsigned long iova;
231+ int rc = 0;
232+
233+ list_for_each_entry(elm, pages_list, next) {
234+ elm->area = kzalloc(sizeof(*elm->area), GFP_KERNEL_ACCOUNT);
235+ if (!elm->area)
236+ return -ENOMEM;
237+ }
238+
239+ down_write(&iopt->iova_rwsem);
240+ if ((length & (iopt->iova_alignment - 1)) || !length) {
241+ rc = -EINVAL;
242+ goto out_unlock;
243+ }
244+
245+ if (flags & IOPT_ALLOC_IOVA) {
246+ /* Use the first entry to guess the ideal IOVA alignment */
247+ elm = list_first_entry(pages_list, struct iopt_pages_list,
248+ next);
249+ rc = iopt_alloc_iova(
250+ iopt, dst_iova,
251+ (uintptr_t)elm->pages->uptr + elm->start_byte, length);
252+ if (rc)
253+ goto out_unlock;
254+ if (IS_ENABLED(CONFIG_IOMMUFD_TEST) &&
255+ WARN_ON(iopt_check_iova(iopt, *dst_iova, length))) {
256+ rc = -EINVAL;
257+ goto out_unlock;
258+ }
259+ } else {
260+ rc = iopt_check_iova(iopt, *dst_iova, length);
261+ if (rc)
262+ goto out_unlock;
263+ }
264+
265+ /*
266+ * Areas are created with a NULL pages so that the IOVA space is
267+ * reserved and we can unlock the iova_rwsem.
268+ */
269+ iova = *dst_iova;
270+ list_for_each_entry(elm, pages_list, next) {
271+ rc = iopt_insert_area(iopt, elm->area, elm->pages, iova,
272+ elm->start_byte, elm->length, iommu_prot);
273+ if (rc)
274+ goto out_unlock;
275+ iova += elm->length;
276+ }
277+
278+out_unlock:
279+ up_write(&iopt->iova_rwsem);
280+ return rc;
281+}
282+
283+static void iopt_abort_area(struct iopt_area *area)
284+{
285+ if (IS_ENABLED(CONFIG_IOMMUFD_TEST))
286+ WARN_ON(area->pages);
287+ if (area->iopt) {
288+ down_write(&area->iopt->iova_rwsem);
289+ interval_tree_remove(&area->node, &area->iopt->area_itree);
290+ up_write(&area->iopt->iova_rwsem);
291+ }
292+ kfree(area);
293+}
294+
295+void iopt_free_pages_list(struct list_head *pages_list)
296+{
297+ struct iopt_pages_list *elm;
298+
299+ while ((elm = list_first_entry_or_null(pages_list,
300+ struct iopt_pages_list, next))) {
301+ if (elm->area)
302+ iopt_abort_area(elm->area);
303+ if (elm->pages)
304+ iopt_put_pages(elm->pages);
305+ list_del(&elm->next);
306+ kfree(elm);
307+ }
308+}
309+
310+static int iopt_fill_domains_pages(struct list_head *pages_list)
311+{
312+ struct iopt_pages_list *undo_elm;
313+ struct iopt_pages_list *elm;
314+ int rc;
315+
316+ list_for_each_entry(elm, pages_list, next) {
317+ rc = iopt_area_fill_domains(elm->area, elm->pages);
318+ if (rc)
319+ goto err_undo;
320+ }
321+ return 0;
322+
323+err_undo:
324+ list_for_each_entry(undo_elm, pages_list, next) {
325+ if (undo_elm == elm)
326+ break;
327+ iopt_area_unfill_domains(undo_elm->area, undo_elm->pages);
328+ }
329+ return rc;
330+}
331+
332+int iopt_map_pages(struct io_pagetable *iopt, struct list_head *pages_list,
333+ unsigned long length, unsigned long *dst_iova,
334+ int iommu_prot, unsigned int flags)
335+{
336+ struct iopt_pages_list *elm;
337+ int rc;
338+
339+ rc = iopt_alloc_area_pages(iopt, pages_list, length, dst_iova,
340+ iommu_prot, flags);
341+ if (rc)
342+ return rc;
343+
344+ down_read(&iopt->domains_rwsem);
345+ rc = iopt_fill_domains_pages(pages_list);
346+ if (rc)
347+ goto out_unlock_domains;
348+
349+ down_write(&iopt->iova_rwsem);
350+ list_for_each_entry(elm, pages_list, next) {
351+ /*
352+ * area->pages must be set inside the domains_rwsem to ensure
353+ * any newly added domains will get filled. Moves the reference
354+ * in from the list.
355+ */
356+ elm->area->pages = elm->pages;
357+ elm->pages = NULL;
358+ elm->area = NULL;
359+ }
360+ up_write(&iopt->iova_rwsem);
361+out_unlock_domains:
362+ up_read(&iopt->domains_rwsem);
363+ return rc;
364+}
365+
366+/**
367+ * iopt_map_user_pages() - Map a user VA to an iova in the io page table
368+ * @ictx: iommufd_ctx the iopt is part of
369+ * @iopt: io_pagetable to act on
370+ * @iova: If IOPT_ALLOC_IOVA is set this is unused on input and contains
371+ * the chosen iova on output. Otherwise is the iova to map to on input
372+ * @uptr: User VA to map
373+ * @length: Number of bytes to map
374+ * @iommu_prot: Combination of IOMMU_READ/WRITE/etc bits for the mapping
375+ * @flags: IOPT_ALLOC_IOVA or zero
376+ *
377+ * iova, uptr, and length must be aligned to iova_alignment. For domain backed
378+ * page tables this will pin the pages and load them into the domain at iova.
379+ * For non-domain page tables this will only setup a lazy reference and the
380+ * caller must use iopt_access_pages() to touch them.
381+ *
382+ * iopt_unmap_iova() must be called to undo this before the io_pagetable can be
383+ * destroyed.
384+ */
385+int iopt_map_user_pages(struct iommufd_ctx *ictx, struct io_pagetable *iopt,
386+ unsigned long *iova, void __user *uptr,
387+ unsigned long length, int iommu_prot,
388+ unsigned int flags)
389+{
390+ struct iopt_pages_list elm = {};
391+ LIST_HEAD(pages_list);
392+ int rc;
393+
394+ elm.pages = iopt_alloc_pages(uptr, length, iommu_prot & IOMMU_WRITE);
395+ if (IS_ERR(elm.pages))
396+ return PTR_ERR(elm.pages);
397+ if (ictx->account_mode == IOPT_PAGES_ACCOUNT_MM &&
398+ elm.pages->account_mode == IOPT_PAGES_ACCOUNT_USER)
399+ elm.pages->account_mode = IOPT_PAGES_ACCOUNT_MM;
400+ elm.start_byte = uptr - elm.pages->uptr;
401+ elm.length = length;
402+ list_add(&elm.next, &pages_list);
403+
404+ rc = iopt_map_pages(iopt, &pages_list, length, iova, iommu_prot, flags);
405+ if (rc) {
406+ if (elm.area)
407+ iopt_abort_area(elm.area);
408+ if (elm.pages)
409+ iopt_put_pages(elm.pages);
410+ return rc;
411+ }
412+ return 0;
413+}
414+
415+int iopt_get_pages(struct io_pagetable *iopt, unsigned long iova,
416+ unsigned long length, struct list_head *pages_list)
417+{
418+ struct iopt_area_contig_iter iter;
419+ unsigned long last_iova;
420+ struct iopt_area *area;
421+ int rc;
422+
423+ if (!length)
424+ return -EINVAL;
425+ if (check_add_overflow(iova, length - 1, &last_iova))
426+ return -EOVERFLOW;
427+
428+ down_read(&iopt->iova_rwsem);
429+ iopt_for_each_contig_area(&iter, area, iopt, iova, last_iova) {
430+ struct iopt_pages_list *elm;
431+ unsigned long last = min(last_iova, iopt_area_last_iova(area));
432+
433+ elm = kzalloc(sizeof(*elm), GFP_KERNEL_ACCOUNT);
434+ if (!elm) {
435+ rc = -ENOMEM;
436+ goto err_free;
437+ }
438+ elm->start_byte = iopt_area_start_byte(area, iter.cur_iova);
439+ elm->pages = area->pages;
440+ elm->length = (last - iter.cur_iova) + 1;
441+ kref_get(&elm->pages->kref);
442+ list_add_tail(&elm->next, pages_list);
443+ }
444+ if (!iopt_area_contig_done(&iter)) {
445+ rc = -ENOENT;
446+ goto err_free;
447+ }
448+ up_read(&iopt->iova_rwsem);
449+ return 0;
450+err_free:
451+ up_read(&iopt->iova_rwsem);
452+ iopt_free_pages_list(pages_list);
453+ return rc;
454+}
455+
456+static int iopt_unmap_iova_range(struct io_pagetable *iopt, unsigned long start,
457+ unsigned long last, unsigned long *unmapped)
458+{
459+ struct iopt_area *area;
460+ unsigned long unmapped_bytes = 0;
461+ int rc = -ENOENT;
462+
463+ /*
464+ * The domains_rwsem must be held in read mode any time any area->pages
465+ * is NULL. This prevents domain attach/detatch from running
466+ * concurrently with cleaning up the area.
467+ */
468+again:
469+ down_read(&iopt->domains_rwsem);
470+ down_write(&iopt->iova_rwsem);
471+ while ((area = iopt_area_iter_first(iopt, start, last))) {
472+ unsigned long area_last = iopt_area_last_iova(area);
473+ unsigned long area_first = iopt_area_iova(area);
474+ struct iopt_pages *pages;
475+
476+ /* Userspace should not race map/unmap's of the same area */
477+ if (!area->pages) {
478+ rc = -EBUSY;
479+ goto out_unlock_iova;
480+ }
481+
482+ if (area_first < start || area_last > last) {
483+ rc = -ENOENT;
484+ goto out_unlock_iova;
485+ }
486+
487+ /*
488+ * num_accesses writers must hold the iova_rwsem too, so we can
489+ * safely read it under the write side of the iovam_rwsem
490+ * without the pages->mutex.
491+ */
492+ if (area->num_accesses) {
493+ start = area_first;
494+ area->prevent_access = true;
495+ up_write(&iopt->iova_rwsem);
496+ up_read(&iopt->domains_rwsem);
497+ iommufd_access_notify_unmap(iopt, area_first,
498+ iopt_area_length(area));
499+ if (WARN_ON(READ_ONCE(area->num_accesses)))
500+ return -EDEADLOCK;
501+ goto again;
502+ }
503+
504+ pages = area->pages;
505+ area->pages = NULL;
506+ up_write(&iopt->iova_rwsem);
507+
508+ iopt_area_unfill_domains(area, pages);
509+ iopt_abort_area(area);
510+ iopt_put_pages(pages);
511+
512+ unmapped_bytes += area_last - area_first + 1;
513+
514+ down_write(&iopt->iova_rwsem);
515+ }
516+ if (unmapped_bytes)
517+ rc = 0;
518+
519+out_unlock_iova:
520+ up_write(&iopt->iova_rwsem);
521+ up_read(&iopt->domains_rwsem);
522+ if (unmapped)
523+ *unmapped = unmapped_bytes;
524+ return rc;
525+}
526+
527+/**
528+ * iopt_unmap_iova() - Remove a range of iova
529+ * @iopt: io_pagetable to act on
530+ * @iova: Starting iova to unmap
531+ * @length: Number of bytes to unmap
532+ * @unmapped: Return number of bytes unmapped
533+ *
534+ * The requested range must be a superset of existing ranges.
535+ * Splitting/truncating IOVA mappings is not allowed.
536+ */
537+int iopt_unmap_iova(struct io_pagetable *iopt, unsigned long iova,
538+ unsigned long length, unsigned long *unmapped)
539+{
540+ unsigned long iova_last;
541+
542+ if (!length)
543+ return -EINVAL;
544+
545+ if (check_add_overflow(iova, length - 1, &iova_last))
546+ return -EOVERFLOW;
547+
548+ return iopt_unmap_iova_range(iopt, iova, iova_last, unmapped);
549+}
550+
551+int iopt_unmap_all(struct io_pagetable *iopt, unsigned long *unmapped)
552+{
553+ int rc;
554+
555+ rc = iopt_unmap_iova_range(iopt, 0, ULONG_MAX, unmapped);
556+ /* If the IOVAs are empty then unmap all succeeds */
557+ if (rc == -ENOENT)
558+ return 0;
559+ return rc;
560+}
561+
562+/* The caller must always free all the nodes in the allowed_iova rb_root. */
563+int iopt_set_allow_iova(struct io_pagetable *iopt,
564+ struct rb_root_cached *allowed_iova)
565+{
566+ struct iopt_allowed *allowed;
567+
568+ down_write(&iopt->iova_rwsem);
569+ swap(*allowed_iova, iopt->allowed_itree);
570+
571+ for (allowed = iopt_allowed_iter_first(iopt, 0, ULONG_MAX); allowed;
572+ allowed = iopt_allowed_iter_next(allowed, 0, ULONG_MAX)) {
573+ if (iopt_reserved_iter_first(iopt, allowed->node.start,
574+ allowed->node.last)) {
575+ swap(*allowed_iova, iopt->allowed_itree);
576+ up_write(&iopt->iova_rwsem);
577+ return -EADDRINUSE;
578+ }
579+ }
580+ up_write(&iopt->iova_rwsem);
581+ return 0;
582+}
583+
584+int iopt_reserve_iova(struct io_pagetable *iopt, unsigned long start,
585+ unsigned long last, void *owner)
586+{
587+ struct iopt_reserved *reserved;
588+
589+ lockdep_assert_held_write(&iopt->iova_rwsem);
590+
591+ if (iopt_area_iter_first(iopt, start, last) ||
592+ iopt_allowed_iter_first(iopt, start, last))
593+ return -EADDRINUSE;
594+
595+ reserved = kzalloc(sizeof(*reserved), GFP_KERNEL_ACCOUNT);
596+ if (!reserved)
597+ return -ENOMEM;
598+ reserved->node.start = start;
599+ reserved->node.last = last;
600+ reserved->owner = owner;
601+ interval_tree_insert(&reserved->node, &iopt->reserved_itree);
602+ return 0;
603+}
604+
605+static void __iopt_remove_reserved_iova(struct io_pagetable *iopt, void *owner)
606+{
607+ struct iopt_reserved *reserved, *next;
608+
609+ lockdep_assert_held_write(&iopt->iova_rwsem);
610+
611+ for (reserved = iopt_reserved_iter_first(iopt, 0, ULONG_MAX); reserved;
612+ reserved = next) {
613+ next = iopt_reserved_iter_next(reserved, 0, ULONG_MAX);
614+
615+ if (reserved->owner == owner) {
616+ interval_tree_remove(&reserved->node,
617+ &iopt->reserved_itree);
618+ kfree(reserved);
619+ }
620+ }
621+}
622+
623+void iopt_remove_reserved_iova(struct io_pagetable *iopt, void *owner)
624+{
625+ down_write(&iopt->iova_rwsem);
626+ __iopt_remove_reserved_iova(iopt, owner);
627+ up_write(&iopt->iova_rwsem);
628+}
629+
630+void iopt_init_table(struct io_pagetable *iopt)
631+{
632+ init_rwsem(&iopt->iova_rwsem);
633+ init_rwsem(&iopt->domains_rwsem);
634+ iopt->area_itree = RB_ROOT_CACHED;
635+ iopt->allowed_itree = RB_ROOT_CACHED;
636+ iopt->reserved_itree = RB_ROOT_CACHED;
637+ xa_init_flags(&iopt->domains, XA_FLAGS_ACCOUNT);
638+ xa_init_flags(&iopt->access_list, XA_FLAGS_ALLOC);
639+
640+ /*
641+ * iopt's start as SW tables that can use the entire size_t IOVA space
642+ * due to the use of size_t in the APIs. They have no alignment
643+ * restriction.
644+ */
645+ iopt->iova_alignment = 1;
646+}
647+
648+void iopt_destroy_table(struct io_pagetable *iopt)
649+{
650+ struct interval_tree_node *node;
651+
652+ if (IS_ENABLED(CONFIG_IOMMUFD_TEST))
653+ iopt_remove_reserved_iova(iopt, NULL);
654+
655+ while ((node = interval_tree_iter_first(&iopt->allowed_itree, 0,
656+ ULONG_MAX))) {
657+ interval_tree_remove(node, &iopt->allowed_itree);
658+ kfree(container_of(node, struct iopt_allowed, node));
659+ }
660+
661+ WARN_ON(!RB_EMPTY_ROOT(&iopt->reserved_itree.rb_root));
662+ WARN_ON(!xa_empty(&iopt->domains));
663+ WARN_ON(!xa_empty(&iopt->access_list));
664+ WARN_ON(!RB_EMPTY_ROOT(&iopt->area_itree.rb_root));
665+}
666+
667+/**
668+ * iopt_unfill_domain() - Unfill a domain with PFNs
669+ * @iopt: io_pagetable to act on
670+ * @domain: domain to unfill
671+ *
672+ * This is used when removing a domain from the iopt. Every area in the iopt
673+ * will be unmapped from the domain. The domain must already be removed from the
674+ * domains xarray.
675+ */
676+static void iopt_unfill_domain(struct io_pagetable *iopt,
677+ struct iommu_domain *domain)
678+{
679+ struct iopt_area *area;
680+
681+ lockdep_assert_held(&iopt->iova_rwsem);
682+ lockdep_assert_held_write(&iopt->domains_rwsem);
683+
684+ /*
685+ * Some other domain is holding all the pfns still, rapidly unmap this
686+ * domain.
687+ */
688+ if (iopt->next_domain_id != 0) {
689+ /* Pick an arbitrary remaining domain to act as storage */
690+ struct iommu_domain *storage_domain =
691+ xa_load(&iopt->domains, 0);
692+
693+ for (area = iopt_area_iter_first(iopt, 0, ULONG_MAX); area;
694+ area = iopt_area_iter_next(area, 0, ULONG_MAX)) {
695+ struct iopt_pages *pages = area->pages;
696+
697+ if (!pages)
698+ continue;
699+
700+ mutex_lock(&pages->mutex);
701+ if (IS_ENABLED(CONFIG_IOMMUFD_TEST))
702+ WARN_ON(!area->storage_domain);
703+ if (area->storage_domain == domain)
704+ area->storage_domain = storage_domain;
705+ mutex_unlock(&pages->mutex);
706+
707+ iopt_area_unmap_domain(area, domain);
708+ }
709+ return;
710+ }
711+
712+ for (area = iopt_area_iter_first(iopt, 0, ULONG_MAX); area;
713+ area = iopt_area_iter_next(area, 0, ULONG_MAX)) {
714+ struct iopt_pages *pages = area->pages;
715+
716+ if (!pages)
717+ continue;
718+
719+ mutex_lock(&pages->mutex);
720+ interval_tree_remove(&area->pages_node, &pages->domains_itree);
721+ WARN_ON(area->storage_domain != domain);
722+ area->storage_domain = NULL;
723+ iopt_area_unfill_domain(area, pages, domain);
724+ mutex_unlock(&pages->mutex);
725+ }
726+}
727+
728+/**
729+ * iopt_fill_domain() - Fill a domain with PFNs
730+ * @iopt: io_pagetable to act on
731+ * @domain: domain to fill
732+ *
733+ * Fill the domain with PFNs from every area in the iopt. On failure the domain
734+ * is left unchanged.
735+ */
736+static int iopt_fill_domain(struct io_pagetable *iopt,
737+ struct iommu_domain *domain)
738+{
739+ struct iopt_area *end_area;
740+ struct iopt_area *area;
741+ int rc;
742+
743+ lockdep_assert_held(&iopt->iova_rwsem);
744+ lockdep_assert_held_write(&iopt->domains_rwsem);
745+
746+ for (area = iopt_area_iter_first(iopt, 0, ULONG_MAX); area;
747+ area = iopt_area_iter_next(area, 0, ULONG_MAX)) {
748+ struct iopt_pages *pages = area->pages;
749+
750+ if (!pages)
751+ continue;
752+
753+ mutex_lock(&pages->mutex);
754+ rc = iopt_area_fill_domain(area, domain);
755+ if (rc) {
756+ mutex_unlock(&pages->mutex);
757+ goto out_unfill;
758+ }
759+ if (!area->storage_domain) {
760+ WARN_ON(iopt->next_domain_id != 0);
761+ area->storage_domain = domain;
762+ interval_tree_insert(&area->pages_node,
763+ &pages->domains_itree);
764+ }
765+ mutex_unlock(&pages->mutex);
766+ }
767+ return 0;
768+
769+out_unfill:
770+ end_area = area;
771+ for (area = iopt_area_iter_first(iopt, 0, ULONG_MAX); area;
772+ area = iopt_area_iter_next(area, 0, ULONG_MAX)) {
773+ struct iopt_pages *pages = area->pages;
774+
775+ if (area == end_area)
776+ break;
777+ if (!pages)
778+ continue;
779+ mutex_lock(&pages->mutex);
780+ if (iopt->next_domain_id == 0) {
781+ interval_tree_remove(&area->pages_node,
782+ &pages->domains_itree);
783+ area->storage_domain = NULL;
784+ }
785+ iopt_area_unfill_domain(area, pages, domain);
786+ mutex_unlock(&pages->mutex);
787+ }
788+ return rc;
789+}
790+
791+/* All existing area's conform to an increased page size */
792+static int iopt_check_iova_alignment(struct io_pagetable *iopt,
793+ unsigned long new_iova_alignment)
794+{
795+ unsigned long align_mask = new_iova_alignment - 1;
796+ struct iopt_area *area;
797+
798+ lockdep_assert_held(&iopt->iova_rwsem);
799+ lockdep_assert_held(&iopt->domains_rwsem);
800+
801+ for (area = iopt_area_iter_first(iopt, 0, ULONG_MAX); area;
802+ area = iopt_area_iter_next(area, 0, ULONG_MAX))
803+ if ((iopt_area_iova(area) & align_mask) ||
804+ (iopt_area_length(area) & align_mask) ||
805+ (area->page_offset & align_mask))
806+ return -EADDRINUSE;
807+
808+ if (IS_ENABLED(CONFIG_IOMMUFD_TEST)) {
809+ struct iommufd_access *access;
810+ unsigned long index;
811+
812+ xa_for_each(&iopt->access_list, index, access)
813+ if (WARN_ON(access->iova_alignment >
814+ new_iova_alignment))
815+ return -EADDRINUSE;
816+ }
817+ return 0;
818+}
819+
820+int iopt_table_add_domain(struct io_pagetable *iopt,
821+ struct iommu_domain *domain)
822+{
823+ const struct iommu_domain_geometry *geometry = &domain->geometry;
824+ struct iommu_domain *iter_domain;
825+ unsigned int new_iova_alignment;
826+ unsigned long index;
827+ int rc;
828+
829+ down_write(&iopt->domains_rwsem);
830+ down_write(&iopt->iova_rwsem);
831+
832+ xa_for_each(&iopt->domains, index, iter_domain) {
833+ if (WARN_ON(iter_domain == domain)) {
834+ rc = -EEXIST;
835+ goto out_unlock;
836+ }
837+ }
838+
839+ /*
840+ * The io page size drives the iova_alignment. Internally the iopt_pages
841+ * works in PAGE_SIZE units and we adjust when mapping sub-PAGE_SIZE
842+ * objects into the iommu_domain.
843+ *
844+ * A iommu_domain must always be able to accept PAGE_SIZE to be
845+ * compatible as we can't guarantee higher contiguity.
846+ */
847+ new_iova_alignment = max_t(unsigned long,
848+ 1UL << __ffs(domain->pgsize_bitmap),
849+ iopt->iova_alignment);
850+ if (new_iova_alignment > PAGE_SIZE) {
851+ rc = -EINVAL;
852+ goto out_unlock;
853+ }
854+ if (new_iova_alignment != iopt->iova_alignment) {
855+ rc = iopt_check_iova_alignment(iopt, new_iova_alignment);
856+ if (rc)
857+ goto out_unlock;
858+ }
859+
860+ /* No area exists that is outside the allowed domain aperture */
861+ if (geometry->aperture_start != 0) {
862+ rc = iopt_reserve_iova(iopt, 0, geometry->aperture_start - 1,
863+ domain);
864+ if (rc)
865+ goto out_reserved;
866+ }
867+ if (geometry->aperture_end != ULONG_MAX) {
868+ rc = iopt_reserve_iova(iopt, geometry->aperture_end + 1,
869+ ULONG_MAX, domain);
870+ if (rc)
871+ goto out_reserved;
872+ }
873+
874+ rc = xa_reserve(&iopt->domains, iopt->next_domain_id, GFP_KERNEL);
875+ if (rc)
876+ goto out_reserved;
877+
878+ rc = iopt_fill_domain(iopt, domain);
879+ if (rc)
880+ goto out_release;
881+
882+ iopt->iova_alignment = new_iova_alignment;
883+ xa_store(&iopt->domains, iopt->next_domain_id, domain, GFP_KERNEL);
884+ iopt->next_domain_id++;
885+ up_write(&iopt->iova_rwsem);
886+ up_write(&iopt->domains_rwsem);
887+ return 0;
888+out_release:
889+ xa_release(&iopt->domains, iopt->next_domain_id);
890+out_reserved:
891+ __iopt_remove_reserved_iova(iopt, domain);
892+out_unlock:
893+ up_write(&iopt->iova_rwsem);
894+ up_write(&iopt->domains_rwsem);
895+ return rc;
896+}
897+
898+static int iopt_calculate_iova_alignment(struct io_pagetable *iopt)
899+{
900+ unsigned long new_iova_alignment;
901+ struct iommufd_access *access;
902+ struct iommu_domain *domain;
903+ unsigned long index;
904+
905+ lockdep_assert_held_write(&iopt->iova_rwsem);
906+ lockdep_assert_held(&iopt->domains_rwsem);
907+
908+ /* See batch_iommu_map_small() */
909+ if (iopt->disable_large_pages)
910+ new_iova_alignment = PAGE_SIZE;
911+ else
912+ new_iova_alignment = 1;
913+
914+ xa_for_each(&iopt->domains, index, domain)
915+ new_iova_alignment = max_t(unsigned long,
916+ 1UL << __ffs(domain->pgsize_bitmap),
917+ new_iova_alignment);
918+ xa_for_each(&iopt->access_list, index, access)
919+ new_iova_alignment = max_t(unsigned long,
920+ access->iova_alignment,
921+ new_iova_alignment);
922+
923+ if (new_iova_alignment > iopt->iova_alignment) {
924+ int rc;
925+
926+ rc = iopt_check_iova_alignment(iopt, new_iova_alignment);
927+ if (rc)
928+ return rc;
929+ }
930+ iopt->iova_alignment = new_iova_alignment;
931+ return 0;
932+}
933+
934+void iopt_table_remove_domain(struct io_pagetable *iopt,
935+ struct iommu_domain *domain)
936+{
937+ struct iommu_domain *iter_domain = NULL;
938+ unsigned long index;
939+
940+ down_write(&iopt->domains_rwsem);
941+ down_write(&iopt->iova_rwsem);
942+
943+ xa_for_each(&iopt->domains, index, iter_domain)
944+ if (iter_domain == domain)
945+ break;
946+ if (WARN_ON(iter_domain != domain) || index >= iopt->next_domain_id)
947+ goto out_unlock;
948+
949+ /*
950+ * Compress the xarray to keep it linear by swapping the entry to erase
951+ * with the tail entry and shrinking the tail.
952+ */
953+ iopt->next_domain_id--;
954+ iter_domain = xa_erase(&iopt->domains, iopt->next_domain_id);
955+ if (index != iopt->next_domain_id)
956+ xa_store(&iopt->domains, index, iter_domain, GFP_KERNEL);
957+
958+ iopt_unfill_domain(iopt, domain);
959+ __iopt_remove_reserved_iova(iopt, domain);
960+
961+ WARN_ON(iopt_calculate_iova_alignment(iopt));
962+out_unlock:
963+ up_write(&iopt->iova_rwsem);
964+ up_write(&iopt->domains_rwsem);
965+}
966+
967+/**
968+ * iopt_area_split - Split an area into two parts at iova
969+ * @area: The area to split
970+ * @iova: Becomes the last of a new area
971+ *
972+ * This splits an area into two. It is part of the VFIO compatibility to allow
973+ * poking a hole in the mapping. The two areas continue to point at the same
974+ * iopt_pages, just with different starting bytes.
975+ */
976+static int iopt_area_split(struct iopt_area *area, unsigned long iova)
977+{
978+ unsigned long alignment = area->iopt->iova_alignment;
979+ unsigned long last_iova = iopt_area_last_iova(area);
980+ unsigned long start_iova = iopt_area_iova(area);
981+ unsigned long new_start = iova + 1;
982+ struct io_pagetable *iopt = area->iopt;
983+ struct iopt_pages *pages = area->pages;
984+ struct iopt_area *lhs;
985+ struct iopt_area *rhs;
986+ int rc;
987+
988+ lockdep_assert_held_write(&iopt->iova_rwsem);
989+
990+ if (iova == start_iova || iova == last_iova)
991+ return 0;
992+
993+ if (!pages || area->prevent_access)
994+ return -EBUSY;
995+
996+ if (new_start & (alignment - 1) ||
997+ iopt_area_start_byte(area, new_start) & (alignment - 1))
998+ return -EINVAL;
999+
1000+ lhs = kzalloc(sizeof(*area), GFP_KERNEL_ACCOUNT);
1001+ if (!lhs)
1002+ return -ENOMEM;
1003+
1004+ rhs = kzalloc(sizeof(*area), GFP_KERNEL_ACCOUNT);
1005+ if (!rhs) {
1006+ rc = -ENOMEM;
1007+ goto err_free_lhs;
1008+ }
1009+
1010+ mutex_lock(&pages->mutex);
1011+ /*
1012+ * Splitting is not permitted if an access exists, we don't track enough
1013+ * information to split existing accesses.
1014+ */
1015+ if (area->num_accesses) {
1016+ rc = -EINVAL;
1017+ goto err_unlock;
1018+ }
1019+
1020+ /*
1021+ * Splitting is not permitted if a domain could have been mapped with
1022+ * huge pages.
1023+ */
1024+ if (area->storage_domain && !iopt->disable_large_pages) {
1025+ rc = -EINVAL;
1026+ goto err_unlock;
1027+ }
1028+
1029+ interval_tree_remove(&area->node, &iopt->area_itree);
1030+ rc = iopt_insert_area(iopt, lhs, area->pages, start_iova,
1031+ iopt_area_start_byte(area, start_iova),
1032+ (new_start - 1) - start_iova + 1,
1033+ area->iommu_prot);
1034+ if (WARN_ON(rc))
1035+ goto err_insert;
1036+
1037+ rc = iopt_insert_area(iopt, rhs, area->pages, new_start,
1038+ iopt_area_start_byte(area, new_start),
1039+ last_iova - new_start + 1, area->iommu_prot);
1040+ if (WARN_ON(rc))
1041+ goto err_remove_lhs;
1042+
1043+ lhs->storage_domain = area->storage_domain;
1044+ lhs->pages = area->pages;
1045+ rhs->storage_domain = area->storage_domain;
1046+ rhs->pages = area->pages;
1047+ kref_get(&rhs->pages->kref);
1048+ kfree(area);
1049+ mutex_unlock(&pages->mutex);
1050+
1051+ /*
1052+ * No change to domains or accesses because the pages hasn't been
1053+ * changed
1054+ */
1055+ return 0;
1056+
1057+err_remove_lhs:
1058+ interval_tree_remove(&lhs->node, &iopt->area_itree);
1059+err_insert:
1060+ interval_tree_insert(&area->node, &iopt->area_itree);
1061+err_unlock:
1062+ mutex_unlock(&pages->mutex);
1063+ kfree(rhs);
1064+err_free_lhs:
1065+ kfree(lhs);
1066+ return rc;
1067+}
1068+
1069+int iopt_cut_iova(struct io_pagetable *iopt, unsigned long *iovas,
1070+ size_t num_iovas)
1071+{
1072+ int rc = 0;
1073+ int i;
1074+
1075+ down_write(&iopt->iova_rwsem);
1076+ for (i = 0; i < num_iovas; i++) {
1077+ struct iopt_area *area;
1078+
1079+ area = iopt_area_iter_first(iopt, iovas[i], iovas[i]);
1080+ if (!area)
1081+ continue;
1082+ rc = iopt_area_split(area, iovas[i]);
1083+ if (rc)
1084+ break;
1085+ }
1086+ up_write(&iopt->iova_rwsem);
1087+ return rc;
1088+}
1089+
1090+void iopt_enable_large_pages(struct io_pagetable *iopt)
1091+{
1092+ int rc;
1093+
1094+ down_write(&iopt->domains_rwsem);
1095+ down_write(&iopt->iova_rwsem);
1096+ WRITE_ONCE(iopt->disable_large_pages, false);
1097+ rc = iopt_calculate_iova_alignment(iopt);
1098+ WARN_ON(rc);
1099+ up_write(&iopt->iova_rwsem);
1100+ up_write(&iopt->domains_rwsem);
1101+}
1102+
1103+int iopt_disable_large_pages(struct io_pagetable *iopt)
1104+{
1105+ int rc = 0;
1106+
1107+ down_write(&iopt->domains_rwsem);
1108+ down_write(&iopt->iova_rwsem);
1109+ if (iopt->disable_large_pages)
1110+ goto out_unlock;
1111+
1112+ /* Won't do it if domains already have pages mapped in them */
1113+ if (!xa_empty(&iopt->domains) &&
1114+ !RB_EMPTY_ROOT(&iopt->area_itree.rb_root)) {
1115+ rc = -EINVAL;
1116+ goto out_unlock;
1117+ }
1118+
1119+ WRITE_ONCE(iopt->disable_large_pages, true);
1120+ rc = iopt_calculate_iova_alignment(iopt);
1121+ if (rc)
1122+ WRITE_ONCE(iopt->disable_large_pages, false);
1123+out_unlock:
1124+ up_write(&iopt->iova_rwsem);
1125+ up_write(&iopt->domains_rwsem);
1126+ return rc;
1127+}
1128+
1129+int iopt_add_access(struct io_pagetable *iopt, struct iommufd_access *access)
1130+{
1131+ int rc;
1132+
1133+ down_write(&iopt->domains_rwsem);
1134+ down_write(&iopt->iova_rwsem);
1135+ rc = xa_alloc(&iopt->access_list, &access->iopt_access_list_id, access,
1136+ xa_limit_16b, GFP_KERNEL_ACCOUNT);
1137+ if (rc)
1138+ goto out_unlock;
1139+
1140+ rc = iopt_calculate_iova_alignment(iopt);
1141+ if (rc) {
1142+ xa_erase(&iopt->access_list, access->iopt_access_list_id);
1143+ goto out_unlock;
1144+ }
1145+
1146+out_unlock:
1147+ up_write(&iopt->iova_rwsem);
1148+ up_write(&iopt->domains_rwsem);
1149+ return rc;
1150+}
1151+
1152+void iopt_remove_access(struct io_pagetable *iopt,
1153+ struct iommufd_access *access)
1154+{
1155+ down_write(&iopt->domains_rwsem);
1156+ down_write(&iopt->iova_rwsem);
1157+ WARN_ON(xa_erase(&iopt->access_list, access->iopt_access_list_id) !=
1158+ access);
1159+ WARN_ON(iopt_calculate_iova_alignment(iopt));
1160+ up_write(&iopt->iova_rwsem);
1161+ up_write(&iopt->domains_rwsem);
1162+}
1163+
1164+/* Narrow the valid_iova_itree to include reserved ranges from a group. */
1165+int iopt_table_enforce_group_resv_regions(struct io_pagetable *iopt,
1166+ struct device *device,
1167+ struct iommu_group *group,
1168+ phys_addr_t *sw_msi_start)
1169+{
1170+ struct iommu_resv_region *resv;
1171+ struct iommu_resv_region *tmp;
1172+ LIST_HEAD(group_resv_regions);
1173+ unsigned int num_hw_msi = 0;
1174+ unsigned int num_sw_msi = 0;
1175+ int rc;
1176+
1177+ down_write(&iopt->iova_rwsem);
1178+ rc = iommu_get_group_resv_regions(group, &group_resv_regions);
1179+ if (rc)
1180+ goto out_unlock;
1181+
1182+ list_for_each_entry(resv, &group_resv_regions, list) {
1183+ if (resv->type == IOMMU_RESV_DIRECT_RELAXABLE)
1184+ continue;
1185+
1186+ if (sw_msi_start && resv->type == IOMMU_RESV_MSI)
1187+ num_hw_msi++;
1188+ if (sw_msi_start && resv->type == IOMMU_RESV_SW_MSI) {
1189+ *sw_msi_start = resv->start;
1190+ num_sw_msi++;
1191+ }
1192+
1193+ rc = iopt_reserve_iova(iopt, resv->start,
1194+ resv->length - 1 + resv->start, device);
1195+ if (rc)
1196+ goto out_reserved;
1197+ }
1198+
1199+ /* Drivers must offer sane combinations of regions */
1200+ if (WARN_ON(num_sw_msi && num_hw_msi) || WARN_ON(num_sw_msi > 1)) {
1201+ rc = -EINVAL;
1202+ goto out_reserved;
1203+ }
1204+
1205+ rc = 0;
1206+ goto out_free_resv;
1207+
1208+out_reserved:
1209+ __iopt_remove_reserved_iova(iopt, device);
1210+out_free_resv:
1211+ list_for_each_entry_safe(resv, tmp, &group_resv_regions, list)
1212+ kfree(resv);
1213+out_unlock:
1214+ up_write(&iopt->iova_rwsem);
1215+ return rc;
1216+}
drivers/iommu/iommufd/io_pagetable.h+241−0
@@ -0,0 +1,241 @@
1+/* SPDX-License-Identifier: GPL-2.0 */
2+/* Copyright (c) 2021-2022, NVIDIA CORPORATION & AFFILIATES.
3+ *
4+ */
5+#ifndef __IO_PAGETABLE_H
6+#define __IO_PAGETABLE_H
7+
8+#include <linux/interval_tree.h>
9+#include <linux/mutex.h>
10+#include <linux/kref.h>
11+#include <linux/xarray.h>
12+
13+#include "iommufd_private.h"
14+
15+struct iommu_domain;
16+
17+/*
18+ * Each io_pagetable is composed of intervals of areas which cover regions of
19+ * the iova that are backed by something. iova not covered by areas is not
20+ * populated in the page table. Each area is fully populated with pages.
21+ *
22+ * iovas are in byte units, but must be iopt->iova_alignment aligned.
23+ *
24+ * pages can be NULL, this means some other thread is still working on setting
25+ * up or tearing down the area. When observed under the write side of the
26+ * domain_rwsem a NULL pages must mean the area is still being setup and no
27+ * domains are filled.
28+ *
29+ * storage_domain points at an arbitrary iommu_domain that is holding the PFNs
30+ * for this area. It is locked by the pages->mutex. This simplifies the locking
31+ * as the pages code can rely on the storage_domain without having to get the
32+ * iopt->domains_rwsem.
33+ *
34+ * The io_pagetable::iova_rwsem protects node
35+ * The iopt_pages::mutex protects pages_node
36+ * iopt and iommu_prot are immutable
37+ * The pages::mutex protects num_accesses
38+ */
39+struct iopt_area {
40+ struct interval_tree_node node;
41+ struct interval_tree_node pages_node;
42+ struct io_pagetable *iopt;
43+ struct iopt_pages *pages;
44+ struct iommu_domain *storage_domain;
45+ /* How many bytes into the first page the area starts */
46+ unsigned int page_offset;
47+ /* IOMMU_READ, IOMMU_WRITE, etc */
48+ int iommu_prot;
49+ bool prevent_access : 1;
50+ unsigned int num_accesses;
51+};
52+
53+struct iopt_allowed {
54+ struct interval_tree_node node;
55+};
56+
57+struct iopt_reserved {
58+ struct interval_tree_node node;
59+ void *owner;
60+};
61+
62+int iopt_area_fill_domains(struct iopt_area *area, struct iopt_pages *pages);
63+void iopt_area_unfill_domains(struct iopt_area *area, struct iopt_pages *pages);
64+
65+int iopt_area_fill_domain(struct iopt_area *area, struct iommu_domain *domain);
66+void iopt_area_unfill_domain(struct iopt_area *area, struct iopt_pages *pages,
67+ struct iommu_domain *domain);
68+void iopt_area_unmap_domain(struct iopt_area *area,
69+ struct iommu_domain *domain);
70+
71+static inline unsigned long iopt_area_index(struct iopt_area *area)
72+{
73+ return area->pages_node.start;
74+}
75+
76+static inline unsigned long iopt_area_last_index(struct iopt_area *area)
77+{
78+ return area->pages_node.last;
79+}
80+
81+static inline unsigned long iopt_area_iova(struct iopt_area *area)
82+{
83+ return area->node.start;
84+}
85+
86+static inline unsigned long iopt_area_last_iova(struct iopt_area *area)
87+{
88+ return area->node.last;
89+}
90+
91+static inline size_t iopt_area_length(struct iopt_area *area)
92+{
93+ return (area->node.last - area->node.start) + 1;
94+}
95+
96+/*
97+ * Number of bytes from the start of the iopt_pages that the iova begins.
98+ * iopt_area_start_byte() / PAGE_SIZE encodes the starting page index
99+ * iopt_area_start_byte() % PAGE_SIZE encodes the offset within that page
100+ */
101+static inline unsigned long iopt_area_start_byte(struct iopt_area *area,
102+ unsigned long iova)
103+{
104+ if (IS_ENABLED(CONFIG_IOMMUFD_TEST))
105+ WARN_ON(iova < iopt_area_iova(area) ||
106+ iova > iopt_area_last_iova(area));
107+ return (iova - iopt_area_iova(area)) + area->page_offset +
108+ iopt_area_index(area) * PAGE_SIZE;
109+}
110+
111+static inline unsigned long iopt_area_iova_to_index(struct iopt_area *area,
112+ unsigned long iova)
113+{
114+ return iopt_area_start_byte(area, iova) / PAGE_SIZE;
115+}
116+
117+#define __make_iopt_iter(name) \
118+ static inline struct iopt_##name *iopt_##name##_iter_first( \
119+ struct io_pagetable *iopt, unsigned long start, \
120+ unsigned long last) \
121+ { \
122+ struct interval_tree_node *node; \
123+ \
124+ lockdep_assert_held(&iopt->iova_rwsem); \
125+ node = interval_tree_iter_first(&iopt->name##_itree, start, \
126+ last); \
127+ if (!node) \
128+ return NULL; \
129+ return container_of(node, struct iopt_##name, node); \
130+ } \
131+ static inline struct iopt_##name *iopt_##name##_iter_next( \
132+ struct iopt_##name *last_node, unsigned long start, \
133+ unsigned long last) \
134+ { \
135+ struct interval_tree_node *node; \
136+ \
137+ node = interval_tree_iter_next(&last_node->node, start, last); \
138+ if (!node) \
139+ return NULL; \
140+ return container_of(node, struct iopt_##name, node); \
141+ }
142+
143+__make_iopt_iter(area)
144+__make_iopt_iter(allowed)
145+__make_iopt_iter(reserved)
146+
147+struct iopt_area_contig_iter {
148+ unsigned long cur_iova;
149+ unsigned long last_iova;
150+ struct iopt_area *area;
151+};
152+struct iopt_area *iopt_area_contig_init(struct iopt_area_contig_iter *iter,
153+ struct io_pagetable *iopt,
154+ unsigned long iova,
155+ unsigned long last_iova);
156+struct iopt_area *iopt_area_contig_next(struct iopt_area_contig_iter *iter);
157+
158+static inline bool iopt_area_contig_done(struct iopt_area_contig_iter *iter)
159+{
160+ return iter->area && iter->last_iova <= iopt_area_last_iova(iter->area);
161+}
162+
163+/*
164+ * Iterate over a contiguous list of areas that span the iova,last_iova range.
165+ * The caller must check iopt_area_contig_done() after the loop to see if
166+ * contiguous areas existed.
167+ */
168+#define iopt_for_each_contig_area(iter, area, iopt, iova, last_iova) \
169+ for (area = iopt_area_contig_init(iter, iopt, iova, last_iova); area; \
170+ area = iopt_area_contig_next(iter))
171+
172+enum {
173+ IOPT_PAGES_ACCOUNT_NONE = 0,
174+ IOPT_PAGES_ACCOUNT_USER = 1,
175+ IOPT_PAGES_ACCOUNT_MM = 2,
176+};
177+
178+/*
179+ * This holds a pinned page list for multiple areas of IO address space. The
180+ * pages always originate from a linear chunk of userspace VA. Multiple
181+ * io_pagetable's, through their iopt_area's, can share a single iopt_pages
182+ * which avoids multi-pinning and double accounting of page consumption.
183+ *
184+ * indexes in this structure are measured in PAGE_SIZE units, are 0 based from
185+ * the start of the uptr and extend to npages. pages are pinned dynamically
186+ * according to the intervals in the access_itree and domains_itree, npinned
187+ * records the current number of pages pinned.
188+ */
189+struct iopt_pages {
190+ struct kref kref;
191+ struct mutex mutex;
192+ size_t npages;
193+ size_t npinned;
194+ size_t last_npinned;
195+ struct task_struct *source_task;
196+ struct mm_struct *source_mm;
197+ struct user_struct *source_user;
198+ void __user *uptr;
199+ bool writable:1;
200+ u8 account_mode;
201+
202+ struct xarray pinned_pfns;
203+ /* Of iopt_pages_access::node */
204+ struct rb_root_cached access_itree;
205+ /* Of iopt_area::pages_node */
206+ struct rb_root_cached domains_itree;
207+};
208+
209+struct iopt_pages *iopt_alloc_pages(void __user *uptr, unsigned long length,
210+ bool writable);
211+void iopt_release_pages(struct kref *kref);
212+static inline void iopt_put_pages(struct iopt_pages *pages)
213+{
214+ kref_put(&pages->kref, iopt_release_pages);
215+}
216+
217+void iopt_pages_fill_from_xarray(struct iopt_pages *pages, unsigned long start,
218+ unsigned long last, struct page **out_pages);
219+int iopt_pages_fill_xarray(struct iopt_pages *pages, unsigned long start,
220+ unsigned long last, struct page **out_pages);
221+void iopt_pages_unfill_xarray(struct iopt_pages *pages, unsigned long start,
222+ unsigned long last);
223+
224+int iopt_area_add_access(struct iopt_area *area, unsigned long start,
225+ unsigned long last, struct page **out_pages,
226+ unsigned int flags);
227+void iopt_area_remove_access(struct iopt_area *area, unsigned long start,
228+ unsigned long last);
229+int iopt_pages_rw_access(struct iopt_pages *pages, unsigned long start_byte,
230+ void *data, unsigned long length, unsigned int flags);
231+
232+/*
233+ * Each interval represents an active iopt_access_pages(), it acts as an
234+ * interval lock that keeps the PFNs pinned and stored in the xarray.
235+ */
236+struct iopt_pages_access {
237+ struct interval_tree_node node;
238+ unsigned int users;
239+};
240+
241+#endif
drivers/iommu/iommufd/ioas.c+398−0
@@ -0,0 +1,398 @@
1+// SPDX-License-Identifier: GPL-2.0-only
2+/*
3+ * Copyright (c) 2021-2022, NVIDIA CORPORATION & AFFILIATES
4+ */
5+#include <linux/interval_tree.h>
6+#include <linux/iommufd.h>
7+#include <linux/iommu.h>
8+#include <uapi/linux/iommufd.h>
9+
10+#include "io_pagetable.h"
11+
12+void iommufd_ioas_destroy(struct iommufd_object *obj)
13+{
14+ struct iommufd_ioas *ioas = container_of(obj, struct iommufd_ioas, obj);
15+ int rc;
16+
17+ rc = iopt_unmap_all(&ioas->iopt, NULL);
18+ WARN_ON(rc && rc != -ENOENT);
19+ iopt_destroy_table(&ioas->iopt);
20+ mutex_destroy(&ioas->mutex);
21+}
22+
23+struct iommufd_ioas *iommufd_ioas_alloc(struct iommufd_ctx *ictx)
24+{
25+ struct iommufd_ioas *ioas;
26+
27+ ioas = iommufd_object_alloc(ictx, ioas, IOMMUFD_OBJ_IOAS);
28+ if (IS_ERR(ioas))
29+ return ioas;
30+
31+ iopt_init_table(&ioas->iopt);
32+ INIT_LIST_HEAD(&ioas->hwpt_list);
33+ mutex_init(&ioas->mutex);
34+ return ioas;
35+}
36+
37+int iommufd_ioas_alloc_ioctl(struct iommufd_ucmd *ucmd)
38+{
39+ struct iommu_ioas_alloc *cmd = ucmd->cmd;
40+ struct iommufd_ioas *ioas;
41+ int rc;
42+
43+ if (cmd->flags)
44+ return -EOPNOTSUPP;
45+
46+ ioas = iommufd_ioas_alloc(ucmd->ictx);
47+ if (IS_ERR(ioas))
48+ return PTR_ERR(ioas);
49+
50+ cmd->out_ioas_id = ioas->obj.id;
51+ rc = iommufd_ucmd_respond(ucmd, sizeof(*cmd));
52+ if (rc)
53+ goto out_table;
54+ iommufd_object_finalize(ucmd->ictx, &ioas->obj);
55+ return 0;
56+
57+out_table:
58+ iommufd_object_abort_and_destroy(ucmd->ictx, &ioas->obj);
59+ return rc;
60+}
61+
62+int iommufd_ioas_iova_ranges(struct iommufd_ucmd *ucmd)
63+{
64+ struct iommu_iova_range __user *ranges;
65+ struct iommu_ioas_iova_ranges *cmd = ucmd->cmd;
66+ struct iommufd_ioas *ioas;
67+ struct interval_tree_span_iter span;
68+ u32 max_iovas;
69+ int rc;
70+
71+ if (cmd->__reserved)
72+ return -EOPNOTSUPP;
73+
74+ ioas = iommufd_get_ioas(ucmd, cmd->ioas_id);
75+ if (IS_ERR(ioas))
76+ return PTR_ERR(ioas);
77+
78+ down_read(&ioas->iopt.iova_rwsem);
79+ max_iovas = cmd->num_iovas;
80+ ranges = u64_to_user_ptr(cmd->allowed_iovas);
81+ cmd->num_iovas = 0;
82+ cmd->out_iova_alignment = ioas->iopt.iova_alignment;
83+ interval_tree_for_each_span(&span, &ioas->iopt.reserved_itree, 0,
84+ ULONG_MAX) {
85+ if (!span.is_hole)
86+ continue;
87+ if (cmd->num_iovas < max_iovas) {
88+ struct iommu_iova_range elm = {
89+ .start = span.start_hole,
90+ .last = span.last_hole,
91+ };
92+
93+ if (copy_to_user(&ranges[cmd->num_iovas], &elm,
94+ sizeof(elm))) {
95+ rc = -EFAULT;
96+ goto out_put;
97+ }
98+ }
99+ cmd->num_iovas++;
100+ }
101+ rc = iommufd_ucmd_respond(ucmd, sizeof(*cmd));
102+ if (rc)
103+ goto out_put;
104+ if (cmd->num_iovas > max_iovas)
105+ rc = -EMSGSIZE;
106+out_put:
107+ up_read(&ioas->iopt.iova_rwsem);
108+ iommufd_put_object(&ioas->obj);
109+ return rc;
110+}
111+
112+static int iommufd_ioas_load_iovas(struct rb_root_cached *itree,
113+ struct iommu_iova_range __user *ranges,
114+ u32 num)
115+{
116+ u32 i;
117+
118+ for (i = 0; i != num; i++) {
119+ struct iommu_iova_range range;
120+ struct iopt_allowed *allowed;
121+
122+ if (copy_from_user(&range, ranges + i, sizeof(range)))
123+ return -EFAULT;
124+
125+ if (range.start >= range.last)
126+ return -EINVAL;
127+
128+ if (interval_tree_iter_first(itree, range.start, range.last))
129+ return -EINVAL;
130+
131+ allowed = kzalloc(sizeof(*allowed), GFP_KERNEL_ACCOUNT);
132+ if (!allowed)
133+ return -ENOMEM;
134+ allowed->node.start = range.start;
135+ allowed->node.last = range.last;
136+
137+ interval_tree_insert(&allowed->node, itree);
138+ }
139+ return 0;
140+}
141+
142+int iommufd_ioas_allow_iovas(struct iommufd_ucmd *ucmd)
143+{
144+ struct iommu_ioas_allow_iovas *cmd = ucmd->cmd;
145+ struct rb_root_cached allowed_iova = RB_ROOT_CACHED;
146+ struct interval_tree_node *node;
147+ struct iommufd_ioas *ioas;
148+ struct io_pagetable *iopt;
149+ int rc = 0;
150+
151+ if (cmd->__reserved)
152+ return -EOPNOTSUPP;
153+
154+ ioas = iommufd_get_ioas(ucmd, cmd->ioas_id);
155+ if (IS_ERR(ioas))
156+ return PTR_ERR(ioas);
157+ iopt = &ioas->iopt;
158+
159+ rc = iommufd_ioas_load_iovas(&allowed_iova,
160+ u64_to_user_ptr(cmd->allowed_iovas),
161+ cmd->num_iovas);
162+ if (rc)
163+ goto out_free;
164+
165+ /*
166+ * We want the allowed tree update to be atomic, so we have to keep the
167+ * original nodes around, and keep track of the new nodes as we allocate
168+ * memory for them. The simplest solution is to have a new/old tree and
169+ * then swap new for old. On success we free the old tree, on failure we
170+ * free the new tree.
171+ */
172+ rc = iopt_set_allow_iova(iopt, &allowed_iova);
173+out_free:
174+ while ((node = interval_tree_iter_first(&allowed_iova, 0, ULONG_MAX))) {
175+ interval_tree_remove(node, &allowed_iova);
176+ kfree(container_of(node, struct iopt_allowed, node));
177+ }
178+ iommufd_put_object(&ioas->obj);
179+ return rc;
180+}
181+
182+static int conv_iommu_prot(u32 map_flags)
183+{
184+ /*
185+ * We provide no manual cache coherency ioctls to userspace and most
186+ * architectures make the CPU ops for cache flushing privileged.
187+ * Therefore we require the underlying IOMMU to support CPU coherent
188+ * operation. Support for IOMMU_CACHE is enforced by the
189+ * IOMMU_CAP_CACHE_COHERENCY test during bind.
190+ */
191+ int iommu_prot = IOMMU_CACHE;
192+
193+ if (map_flags & IOMMU_IOAS_MAP_WRITEABLE)
194+ iommu_prot |= IOMMU_WRITE;
195+ if (map_flags & IOMMU_IOAS_MAP_READABLE)
196+ iommu_prot |= IOMMU_READ;
197+ return iommu_prot;
198+}
199+
200+int iommufd_ioas_map(struct iommufd_ucmd *ucmd)
201+{
202+ struct iommu_ioas_map *cmd = ucmd->cmd;
203+ unsigned long iova = cmd->iova;
204+ struct iommufd_ioas *ioas;
205+ unsigned int flags = 0;
206+ int rc;
207+
208+ if ((cmd->flags &
209+ ~(IOMMU_IOAS_MAP_FIXED_IOVA | IOMMU_IOAS_MAP_WRITEABLE |
210+ IOMMU_IOAS_MAP_READABLE)) ||
211+ cmd->__reserved)
212+ return -EOPNOTSUPP;
213+ if (cmd->iova >= ULONG_MAX || cmd->length >= ULONG_MAX)
214+ return -EOVERFLOW;
215+
216+ ioas = iommufd_get_ioas(ucmd, cmd->ioas_id);
217+ if (IS_ERR(ioas))
218+ return PTR_ERR(ioas);
219+
220+ if (!(cmd->flags & IOMMU_IOAS_MAP_FIXED_IOVA))
221+ flags = IOPT_ALLOC_IOVA;
222+ rc = iopt_map_user_pages(ucmd->ictx, &ioas->iopt, &iova,
223+ u64_to_user_ptr(cmd->user_va), cmd->length,
224+ conv_iommu_prot(cmd->flags), flags);
225+ if (rc)
226+ goto out_put;
227+
228+ cmd->iova = iova;
229+ rc = iommufd_ucmd_respond(ucmd, sizeof(*cmd));
230+out_put:
231+ iommufd_put_object(&ioas->obj);
232+ return rc;
233+}
234+
235+int iommufd_ioas_copy(struct iommufd_ucmd *ucmd)
236+{
237+ struct iommu_ioas_copy *cmd = ucmd->cmd;
238+ struct iommufd_ioas *src_ioas;
239+ struct iommufd_ioas *dst_ioas;
240+ unsigned int flags = 0;
241+ LIST_HEAD(pages_list);
242+ unsigned long iova;
243+ int rc;
244+
245+ iommufd_test_syz_conv_iova_id(ucmd, cmd->src_ioas_id, &cmd->src_iova,
246+ &cmd->flags);
247+
248+ if ((cmd->flags &
249+ ~(IOMMU_IOAS_MAP_FIXED_IOVA | IOMMU_IOAS_MAP_WRITEABLE |
250+ IOMMU_IOAS_MAP_READABLE)))
251+ return -EOPNOTSUPP;
252+ if (cmd->length >= ULONG_MAX || cmd->src_iova >= ULONG_MAX ||
253+ cmd->dst_iova >= ULONG_MAX)
254+ return -EOVERFLOW;
255+
256+ src_ioas = iommufd_get_ioas(ucmd, cmd->src_ioas_id);
257+ if (IS_ERR(src_ioas))
258+ return PTR_ERR(src_ioas);
259+ rc = iopt_get_pages(&src_ioas->iopt, cmd->src_iova, cmd->length,
260+ &pages_list);
261+ iommufd_put_object(&src_ioas->obj);
262+ if (rc)
263+ return rc;
264+
265+ dst_ioas = iommufd_get_ioas(ucmd, cmd->dst_ioas_id);
266+ if (IS_ERR(dst_ioas)) {
267+ rc = PTR_ERR(dst_ioas);
268+ goto out_pages;
269+ }
270+
271+ if (!(cmd->flags & IOMMU_IOAS_MAP_FIXED_IOVA))
272+ flags = IOPT_ALLOC_IOVA;
273+ iova = cmd->dst_iova;
274+ rc = iopt_map_pages(&dst_ioas->iopt, &pages_list, cmd->length, &iova,
275+ conv_iommu_prot(cmd->flags), flags);
276+ if (rc)
277+ goto out_put_dst;
278+
279+ cmd->dst_iova = iova;
280+ rc = iommufd_ucmd_respond(ucmd, sizeof(*cmd));
281+out_put_dst:
282+ iommufd_put_object(&dst_ioas->obj);
283+out_pages:
284+ iopt_free_pages_list(&pages_list);
285+ return rc;
286+}
287+
288+int iommufd_ioas_unmap(struct iommufd_ucmd *ucmd)
289+{
290+ struct iommu_ioas_unmap *cmd = ucmd->cmd;
291+ struct iommufd_ioas *ioas;
292+ unsigned long unmapped = 0;
293+ int rc;
294+
295+ ioas = iommufd_get_ioas(ucmd, cmd->ioas_id);
296+ if (IS_ERR(ioas))
297+ return PTR_ERR(ioas);
298+
299+ if (cmd->iova == 0 && cmd->length == U64_MAX) {
300+ rc = iopt_unmap_all(&ioas->iopt, &unmapped);
301+ if (rc)
302+ goto out_put;
303+ } else {
304+ if (cmd->iova >= ULONG_MAX || cmd->length >= ULONG_MAX) {
305+ rc = -EOVERFLOW;
306+ goto out_put;
307+ }
308+ rc = iopt_unmap_iova(&ioas->iopt, cmd->iova, cmd->length,
309+ &unmapped);
310+ if (rc)
311+ goto out_put;
312+ }
313+
314+ cmd->length = unmapped;
315+ rc = iommufd_ucmd_respond(ucmd, sizeof(*cmd));
316+
317+out_put:
318+ iommufd_put_object(&ioas->obj);
319+ return rc;
320+}
321+
322+int iommufd_option_rlimit_mode(struct iommu_option *cmd,
323+ struct iommufd_ctx *ictx)
324+{
325+ if (cmd->object_id)
326+ return -EOPNOTSUPP;
327+
328+ if (cmd->op == IOMMU_OPTION_OP_GET) {
329+ cmd->val64 = ictx->account_mode == IOPT_PAGES_ACCOUNT_MM;
330+ return 0;
331+ }
332+ if (cmd->op == IOMMU_OPTION_OP_SET) {
333+ int rc = 0;
334+
335+ if (!capable(CAP_SYS_RESOURCE))
336+ return -EPERM;
337+
338+ xa_lock(&ictx->objects);
339+ if (!xa_empty(&ictx->objects)) {
340+ rc = -EBUSY;
341+ } else {
342+ if (cmd->val64 == 0)
343+ ictx->account_mode = IOPT_PAGES_ACCOUNT_USER;
344+ else if (cmd->val64 == 1)
345+ ictx->account_mode = IOPT_PAGES_ACCOUNT_MM;
346+ else
347+ rc = -EINVAL;
348+ }
349+ xa_unlock(&ictx->objects);
350+
351+ return rc;
352+ }
353+ return -EOPNOTSUPP;
354+}
355+
356+static int iommufd_ioas_option_huge_pages(struct iommu_option *cmd,
357+ struct iommufd_ioas *ioas)
358+{
359+ if (cmd->op == IOMMU_OPTION_OP_GET) {
360+ cmd->val64 = !ioas->iopt.disable_large_pages;
361+ return 0;
362+ }
363+ if (cmd->op == IOMMU_OPTION_OP_SET) {
364+ if (cmd->val64 == 0)
365+ return iopt_disable_large_pages(&ioas->iopt);
366+ if (cmd->val64 == 1) {
367+ iopt_enable_large_pages(&ioas->iopt);
368+ return 0;
369+ }
370+ return -EINVAL;
371+ }
372+ return -EOPNOTSUPP;
373+}
374+
375+int iommufd_ioas_option(struct iommufd_ucmd *ucmd)
376+{
377+ struct iommu_option *cmd = ucmd->cmd;
378+ struct iommufd_ioas *ioas;
379+ int rc = 0;
380+
381+ if (cmd->__reserved)
382+ return -EOPNOTSUPP;
383+
384+ ioas = iommufd_get_ioas(ucmd, cmd->object_id);
385+ if (IS_ERR(ioas))
386+ return PTR_ERR(ioas);
387+
388+ switch (cmd->option_id) {
389+ case IOMMU_OPTION_HUGE_PAGES:
390+ rc = iommufd_ioas_option_huge_pages(cmd, ioas);
391+ break;
392+ default:
393+ rc = -EOPNOTSUPP;
394+ }
395+
396+ iommufd_put_object(&ioas->obj);
397+ return rc;
398+}
drivers/iommu/iommufd/iommufd_private.h+307−0
@@ -0,0 +1,307 @@
1+/* SPDX-License-Identifier: GPL-2.0-only */
2+/* Copyright (c) 2021-2022, NVIDIA CORPORATION & AFFILIATES
3+ */
4+#ifndef __IOMMUFD_PRIVATE_H
5+#define __IOMMUFD_PRIVATE_H
6+
7+#include <linux/rwsem.h>
8+#include <linux/xarray.h>
9+#include <linux/refcount.h>
10+#include <linux/uaccess.h>
11+
12+struct iommu_domain;
13+struct iommu_group;
14+struct iommu_option;
15+
16+struct iommufd_ctx {
17+ struct file *file;
18+ struct xarray objects;
19+
20+ u8 account_mode;
21+ struct iommufd_ioas *vfio_ioas;
22+};
23+
24+/*
25+ * The IOVA to PFN map. The map automatically copies the PFNs into multiple
26+ * domains and permits sharing of PFNs between io_pagetable instances. This
27+ * supports both a design where IOAS's are 1:1 with a domain (eg because the
28+ * domain is HW customized), or where the IOAS is 1:N with multiple generic
29+ * domains. The io_pagetable holds an interval tree of iopt_areas which point
30+ * to shared iopt_pages which hold the pfns mapped to the page table.
31+ *
32+ * The locking order is domains_rwsem -> iova_rwsem -> pages::mutex
33+ */
34+struct io_pagetable {
35+ struct rw_semaphore domains_rwsem;
36+ struct xarray domains;
37+ struct xarray access_list;
38+ unsigned int next_domain_id;
39+
40+ struct rw_semaphore iova_rwsem;
41+ struct rb_root_cached area_itree;
42+ /* IOVA that cannot become reserved, struct iopt_allowed */
43+ struct rb_root_cached allowed_itree;
44+ /* IOVA that cannot be allocated, struct iopt_reserved */
45+ struct rb_root_cached reserved_itree;
46+ u8 disable_large_pages;
47+ unsigned long iova_alignment;
48+};
49+
50+void iopt_init_table(struct io_pagetable *iopt);
51+void iopt_destroy_table(struct io_pagetable *iopt);
52+int iopt_get_pages(struct io_pagetable *iopt, unsigned long iova,
53+ unsigned long length, struct list_head *pages_list);
54+void iopt_free_pages_list(struct list_head *pages_list);
55+enum {
56+ IOPT_ALLOC_IOVA = 1 << 0,
57+};
58+int iopt_map_user_pages(struct iommufd_ctx *ictx, struct io_pagetable *iopt,
59+ unsigned long *iova, void __user *uptr,
60+ unsigned long length, int iommu_prot,
61+ unsigned int flags);
62+int iopt_map_pages(struct io_pagetable *iopt, struct list_head *pages_list,
63+ unsigned long length, unsigned long *dst_iova,
64+ int iommu_prot, unsigned int flags);
65+int iopt_unmap_iova(struct io_pagetable *iopt, unsigned long iova,
66+ unsigned long length, unsigned long *unmapped);
67+int iopt_unmap_all(struct io_pagetable *iopt, unsigned long *unmapped);
68+
69+void iommufd_access_notify_unmap(struct io_pagetable *iopt, unsigned long iova,
70+ unsigned long length);
71+int iopt_table_add_domain(struct io_pagetable *iopt,
72+ struct iommu_domain *domain);
73+void iopt_table_remove_domain(struct io_pagetable *iopt,
74+ struct iommu_domain *domain);
75+int iopt_table_enforce_group_resv_regions(struct io_pagetable *iopt,
76+ struct device *device,
77+ struct iommu_group *group,
78+ phys_addr_t *sw_msi_start);
79+int iopt_set_allow_iova(struct io_pagetable *iopt,
80+ struct rb_root_cached *allowed_iova);
81+int iopt_reserve_iova(struct io_pagetable *iopt, unsigned long start,
82+ unsigned long last, void *owner);
83+void iopt_remove_reserved_iova(struct io_pagetable *iopt, void *owner);
84+int iopt_cut_iova(struct io_pagetable *iopt, unsigned long *iovas,
85+ size_t num_iovas);
86+void iopt_enable_large_pages(struct io_pagetable *iopt);
87+int iopt_disable_large_pages(struct io_pagetable *iopt);
88+
89+struct iommufd_ucmd {
90+ struct iommufd_ctx *ictx;
91+ void __user *ubuffer;
92+ u32 user_size;
93+ void *cmd;
94+};
95+
96+int iommufd_vfio_ioctl(struct iommufd_ctx *ictx, unsigned int cmd,
97+ unsigned long arg);
98+
99+/* Copy the response in ucmd->cmd back to userspace. */
100+static inline int iommufd_ucmd_respond(struct iommufd_ucmd *ucmd,
101+ size_t cmd_len)
102+{
103+ if (copy_to_user(ucmd->ubuffer, ucmd->cmd,
104+ min_t(size_t, ucmd->user_size, cmd_len)))
105+ return -EFAULT;
106+ return 0;
107+}
108+
109+enum iommufd_object_type {
110+ IOMMUFD_OBJ_NONE,
111+ IOMMUFD_OBJ_ANY = IOMMUFD_OBJ_NONE,
112+ IOMMUFD_OBJ_DEVICE,
113+ IOMMUFD_OBJ_HW_PAGETABLE,
114+ IOMMUFD_OBJ_IOAS,
115+ IOMMUFD_OBJ_ACCESS,
116+#ifdef CONFIG_IOMMUFD_TEST
117+ IOMMUFD_OBJ_SELFTEST,
118+#endif
119+};
120+
121+/* Base struct for all objects with a userspace ID handle. */
122+struct iommufd_object {
123+ struct rw_semaphore destroy_rwsem;
124+ refcount_t users;
125+ enum iommufd_object_type type;
126+ unsigned int id;
127+};
128+
129+static inline bool iommufd_lock_obj(struct iommufd_object *obj)
130+{
131+ if (!down_read_trylock(&obj->destroy_rwsem))
132+ return false;
133+ if (!refcount_inc_not_zero(&obj->users)) {
134+ up_read(&obj->destroy_rwsem);
135+ return false;
136+ }
137+ return true;
138+}
139+
140+struct iommufd_object *iommufd_get_object(struct iommufd_ctx *ictx, u32 id,
141+ enum iommufd_object_type type);
142+static inline void iommufd_put_object(struct iommufd_object *obj)
143+{
144+ refcount_dec(&obj->users);
145+ up_read(&obj->destroy_rwsem);
146+}
147+
148+/**
149+ * iommufd_ref_to_users() - Switch from destroy_rwsem to users refcount
150+ * protection
151+ * @obj - Object to release
152+ *
153+ * Objects have two refcount protections (destroy_rwsem and the refcount_t
154+ * users). Holding either of these will prevent the object from being destroyed.
155+ *
156+ * Depending on the use case, one protection or the other is appropriate. In
157+ * most cases references are being protected by the destroy_rwsem. This allows
158+ * orderly destruction of the object because iommufd_object_destroy_user() will
159+ * wait for it to become unlocked. However, as a rwsem, it cannot be held across
160+ * a system call return. So cases that have longer term needs must switch
161+ * to the weaker users refcount_t.
162+ *
163+ * With users protection iommufd_object_destroy_user() will return false,
164+ * refusing to destroy the object, causing -EBUSY to userspace.
165+ */
166+static inline void iommufd_ref_to_users(struct iommufd_object *obj)
167+{
168+ up_read(&obj->destroy_rwsem);
169+ /* iommufd_lock_obj() obtains users as well */
170+}
171+void iommufd_object_abort(struct iommufd_ctx *ictx, struct iommufd_object *obj);
172+void iommufd_object_abort_and_destroy(struct iommufd_ctx *ictx,
173+ struct iommufd_object *obj);
174+void iommufd_object_finalize(struct iommufd_ctx *ictx,
175+ struct iommufd_object *obj);
176+bool iommufd_object_destroy_user(struct iommufd_ctx *ictx,
177+ struct iommufd_object *obj);
178+struct iommufd_object *_iommufd_object_alloc(struct iommufd_ctx *ictx,
179+ size_t size,
180+ enum iommufd_object_type type);
181+
182+#define iommufd_object_alloc(ictx, ptr, type) \
183+ container_of(_iommufd_object_alloc( \
184+ ictx, \
185+ sizeof(*(ptr)) + BUILD_BUG_ON_ZERO( \
186+ offsetof(typeof(*(ptr)), \
187+ obj) != 0), \
188+ type), \
189+ typeof(*(ptr)), obj)
190+
191+/*
192+ * The IO Address Space (IOAS) pagetable is a virtual page table backed by the
193+ * io_pagetable object. It is a user controlled mapping of IOVA -> PFNs. The
194+ * mapping is copied into all of the associated domains and made available to
195+ * in-kernel users.
196+ *
197+ * Every iommu_domain that is created is wrapped in a iommufd_hw_pagetable
198+ * object. When we go to attach a device to an IOAS we need to get an
199+ * iommu_domain and wrapping iommufd_hw_pagetable for it.
200+ *
201+ * An iommu_domain & iommfd_hw_pagetable will be automatically selected
202+ * for a device based on the hwpt_list. If no suitable iommu_domain
203+ * is found a new iommu_domain will be created.
204+ */
205+struct iommufd_ioas {
206+ struct iommufd_object obj;
207+ struct io_pagetable iopt;
208+ struct mutex mutex;
209+ struct list_head hwpt_list;
210+};
211+
212+static inline struct iommufd_ioas *iommufd_get_ioas(struct iommufd_ucmd *ucmd,
213+ u32 id)
214+{
215+ return container_of(iommufd_get_object(ucmd->ictx, id,
216+ IOMMUFD_OBJ_IOAS),
217+ struct iommufd_ioas, obj);
218+}
219+
220+struct iommufd_ioas *iommufd_ioas_alloc(struct iommufd_ctx *ictx);
221+int iommufd_ioas_alloc_ioctl(struct iommufd_ucmd *ucmd);
222+void iommufd_ioas_destroy(struct iommufd_object *obj);
223+int iommufd_ioas_iova_ranges(struct iommufd_ucmd *ucmd);
224+int iommufd_ioas_allow_iovas(struct iommufd_ucmd *ucmd);
225+int iommufd_ioas_map(struct iommufd_ucmd *ucmd);
226+int iommufd_ioas_copy(struct iommufd_ucmd *ucmd);
227+int iommufd_ioas_unmap(struct iommufd_ucmd *ucmd);
228+int iommufd_ioas_option(struct iommufd_ucmd *ucmd);
229+int iommufd_option_rlimit_mode(struct iommu_option *cmd,
230+ struct iommufd_ctx *ictx);
231+
232+int iommufd_vfio_ioas(struct iommufd_ucmd *ucmd);
233+
234+/*
235+ * A HW pagetable is called an iommu_domain inside the kernel. This user object
236+ * allows directly creating and inspecting the domains. Domains that have kernel
237+ * owned page tables will be associated with an iommufd_ioas that provides the
238+ * IOVA to PFN map.
239+ */
240+struct iommufd_hw_pagetable {
241+ struct iommufd_object obj;
242+ struct iommufd_ioas *ioas;
243+ struct iommu_domain *domain;
244+ bool auto_domain : 1;
245+ bool enforce_cache_coherency : 1;
246+ bool msi_cookie : 1;
247+ /* Head at iommufd_ioas::hwpt_list */
248+ struct list_head hwpt_item;
249+ struct mutex devices_lock;
250+ struct list_head devices;
251+};
252+
253+struct iommufd_hw_pagetable *
254+iommufd_hw_pagetable_alloc(struct iommufd_ctx *ictx, struct iommufd_ioas *ioas,
255+ struct device *dev);
256+void iommufd_hw_pagetable_destroy(struct iommufd_object *obj);
257+
258+void iommufd_device_destroy(struct iommufd_object *obj);
259+
260+struct iommufd_access {
261+ struct iommufd_object obj;
262+ struct iommufd_ctx *ictx;
263+ struct iommufd_ioas *ioas;
264+ const struct iommufd_access_ops *ops;
265+ void *data;
266+ unsigned long iova_alignment;
267+ u32 iopt_access_list_id;
268+};
269+
270+int iopt_add_access(struct io_pagetable *iopt, struct iommufd_access *access);
271+void iopt_remove_access(struct io_pagetable *iopt,
272+ struct iommufd_access *access);
273+void iommufd_access_destroy_object(struct iommufd_object *obj);
274+
275+#ifdef CONFIG_IOMMUFD_TEST
276+struct iommufd_hw_pagetable *
277+iommufd_device_selftest_attach(struct iommufd_ctx *ictx,
278+ struct iommufd_ioas *ioas,
279+ struct device *mock_dev);
280+void iommufd_device_selftest_detach(struct iommufd_ctx *ictx,
281+ struct iommufd_hw_pagetable *hwpt);
282+int iommufd_test(struct iommufd_ucmd *ucmd);
283+void iommufd_selftest_destroy(struct iommufd_object *obj);
284+extern size_t iommufd_test_memory_limit;
285+void iommufd_test_syz_conv_iova_id(struct iommufd_ucmd *ucmd,
286+ unsigned int ioas_id, u64 *iova, u32 *flags);
287+bool iommufd_should_fail(void);
288+void __init iommufd_test_init(void);
289+void iommufd_test_exit(void);
290+#else
291+static inline void iommufd_test_syz_conv_iova_id(struct iommufd_ucmd *ucmd,
292+ unsigned int ioas_id,
293+ u64 *iova, u32 *flags)
294+{
295+}
296+static inline bool iommufd_should_fail(void)
297+{
298+ return false;
299+}
300+static inline void __init iommufd_test_init(void)
301+{
302+}
303+static inline void iommufd_test_exit(void)
304+{
305+}
306+#endif
307+#endif
drivers/iommu/iommufd/iommufd_test.h+93−0
@@ -0,0 +1,93 @@
1+/* SPDX-License-Identifier: GPL-2.0 */
2+/* Copyright (c) 2021-2022, NVIDIA CORPORATION & AFFILIATES.
3+ */
4+#ifndef _UAPI_IOMMUFD_TEST_H
5+#define _UAPI_IOMMUFD_TEST_H
6+
7+#include <linux/types.h>
8+#include <linux/iommufd.h>
9+
10+enum {
11+ IOMMU_TEST_OP_ADD_RESERVED = 1,
12+ IOMMU_TEST_OP_MOCK_DOMAIN,
13+ IOMMU_TEST_OP_MD_CHECK_MAP,
14+ IOMMU_TEST_OP_MD_CHECK_REFS,
15+ IOMMU_TEST_OP_CREATE_ACCESS,
16+ IOMMU_TEST_OP_DESTROY_ACCESS_PAGES,
17+ IOMMU_TEST_OP_ACCESS_PAGES,
18+ IOMMU_TEST_OP_ACCESS_RW,
19+ IOMMU_TEST_OP_SET_TEMP_MEMORY_LIMIT,
20+};
21+
22+enum {
23+ MOCK_APERTURE_START = 1UL << 24,
24+ MOCK_APERTURE_LAST = (1UL << 31) - 1,
25+};
26+
27+enum {
28+ MOCK_FLAGS_ACCESS_WRITE = 1 << 0,
29+ MOCK_FLAGS_ACCESS_SYZ = 1 << 16,
30+};
31+
32+enum {
33+ MOCK_ACCESS_RW_WRITE = 1 << 0,
34+ MOCK_ACCESS_RW_SLOW_PATH = 1 << 2,
35+};
36+
37+enum {
38+ MOCK_FLAGS_ACCESS_CREATE_NEEDS_PIN_PAGES = 1 << 0,
39+};
40+
41+struct iommu_test_cmd {
42+ __u32 size;
43+ __u32 op;
44+ __u32 id;
45+ __u32 __reserved;
46+ union {
47+ struct {
48+ __aligned_u64 start;
49+ __aligned_u64 length;
50+ } add_reserved;
51+ struct {
52+ __u32 out_device_id;
53+ __u32 out_hwpt_id;
54+ } mock_domain;
55+ struct {
56+ __aligned_u64 iova;
57+ __aligned_u64 length;
58+ __aligned_u64 uptr;
59+ } check_map;
60+ struct {
61+ __aligned_u64 length;
62+ __aligned_u64 uptr;
63+ __u32 refs;
64+ } check_refs;
65+ struct {
66+ __u32 out_access_fd;
67+ __u32 flags;
68+ } create_access;
69+ struct {
70+ __u32 access_pages_id;
71+ } destroy_access_pages;
72+ struct {
73+ __u32 flags;
74+ __u32 out_access_pages_id;
75+ __aligned_u64 iova;
76+ __aligned_u64 length;
77+ __aligned_u64 uptr;
78+ } access_pages;
79+ struct {
80+ __aligned_u64 iova;
81+ __aligned_u64 length;
82+ __aligned_u64 uptr;
83+ __u32 flags;
84+ } access_rw;
85+ struct {
86+ __u32 limit;
87+ } memory_limit;
88+ };
89+ __u32 last;
90+};
91+#define IOMMU_TEST_CMD _IO(IOMMUFD_TYPE, IOMMUFD_CMD_BASE + 32)
92+
93+#endif
drivers/iommu/iommufd/main.c+460−0
@@ -0,0 +1,460 @@
1+// SPDX-License-Identifier: GPL-2.0-only
2+/* Copyright (C) 2021 Intel Corporation
3+ * Copyright (c) 2021-2022, NVIDIA CORPORATION & AFFILIATES
4+ *
5+ * iommufd provides control over the IOMMU HW objects created by IOMMU kernel
6+ * drivers. IOMMU HW objects revolve around IO page tables that map incoming DMA
7+ * addresses (IOVA) to CPU addresses.
8+ */
9+#define pr_fmt(fmt) "iommufd: " fmt
10+
11+#include <linux/file.h>
12+#include <linux/fs.h>
13+#include <linux/module.h>
14+#include <linux/slab.h>
15+#include <linux/miscdevice.h>
16+#include <linux/mutex.h>
17+#include <linux/bug.h>
18+#include <uapi/linux/iommufd.h>
19+#include <linux/iommufd.h>
20+
21+#include "io_pagetable.h"
22+#include "iommufd_private.h"
23+#include "iommufd_test.h"
24+
25+struct iommufd_object_ops {
26+ void (*destroy)(struct iommufd_object *obj);
27+};
28+static const struct iommufd_object_ops iommufd_object_ops[];
29+static struct miscdevice vfio_misc_dev;
30+
31+struct iommufd_object *_iommufd_object_alloc(struct iommufd_ctx *ictx,
32+ size_t size,
33+ enum iommufd_object_type type)
34+{
35+ struct iommufd_object *obj;
36+ int rc;
37+
38+ obj = kzalloc(size, GFP_KERNEL_ACCOUNT);
39+ if (!obj)
40+ return ERR_PTR(-ENOMEM);
41+ obj->type = type;
42+ init_rwsem(&obj->destroy_rwsem);
43+ refcount_set(&obj->users, 1);
44+
45+ /*
46+ * Reserve an ID in the xarray but do not publish the pointer yet since
47+ * the caller hasn't initialized it yet. Once the pointer is published
48+ * in the xarray and visible to other threads we can't reliably destroy
49+ * it anymore, so the caller must complete all errorable operations
50+ * before calling iommufd_object_finalize().
51+ */
52+ rc = xa_alloc(&ictx->objects, &obj->id, XA_ZERO_ENTRY,
53+ xa_limit_32b, GFP_KERNEL_ACCOUNT);
54+ if (rc)
55+ goto out_free;
56+ return obj;
57+out_free:
58+ kfree(obj);
59+ return ERR_PTR(rc);
60+}
61+
62+/*
63+ * Allow concurrent access to the object.
64+ *
65+ * Once another thread can see the object pointer it can prevent object
66+ * destruction. Expect for special kernel-only objects there is no in-kernel way
67+ * to reliably destroy a single object. Thus all APIs that are creating objects
68+ * must use iommufd_object_abort() to handle their errors and only call
69+ * iommufd_object_finalize() once object creation cannot fail.
70+ */
71+void iommufd_object_finalize(struct iommufd_ctx *ictx,
72+ struct iommufd_object *obj)
73+{
74+ void *old;
75+
76+ old = xa_store(&ictx->objects, obj->id, obj, GFP_KERNEL);
77+ /* obj->id was returned from xa_alloc() so the xa_store() cannot fail */
78+ WARN_ON(old);
79+}
80+
81+/* Undo _iommufd_object_alloc() if iommufd_object_finalize() was not called */
82+void iommufd_object_abort(struct iommufd_ctx *ictx, struct iommufd_object *obj)
83+{
84+ void *old;
85+
86+ old = xa_erase(&ictx->objects, obj->id);
87+ WARN_ON(old);
88+ kfree(obj);
89+}
90+
91+/*
92+ * Abort an object that has been fully initialized and needs destroy, but has
93+ * not been finalized.
94+ */
95+void iommufd_object_abort_and_destroy(struct iommufd_ctx *ictx,
96+ struct iommufd_object *obj)
97+{
98+ iommufd_object_ops[obj->type].destroy(obj);
99+ iommufd_object_abort(ictx, obj);
100+}
101+
102+struct iommufd_object *iommufd_get_object(struct iommufd_ctx *ictx, u32 id,
103+ enum iommufd_object_type type)
104+{
105+ struct iommufd_object *obj;
106+
107+ if (iommufd_should_fail())
108+ return ERR_PTR(-ENOENT);
109+
110+ xa_lock(&ictx->objects);
111+ obj = xa_load(&ictx->objects, id);
112+ if (!obj || (type != IOMMUFD_OBJ_ANY && obj->type != type) ||
113+ !iommufd_lock_obj(obj))
114+ obj = ERR_PTR(-ENOENT);
115+ xa_unlock(&ictx->objects);
116+ return obj;
117+}
118+
119+/*
120+ * The caller holds a users refcount and wants to destroy the object. Returns
121+ * true if the object was destroyed. In all cases the caller no longer has a
122+ * reference on obj.
123+ */
124+bool iommufd_object_destroy_user(struct iommufd_ctx *ictx,
125+ struct iommufd_object *obj)
126+{
127+ /*
128+ * The purpose of the destroy_rwsem is to ensure deterministic
129+ * destruction of objects used by external drivers and destroyed by this
130+ * function. Any temporary increment of the refcount must hold the read
131+ * side of this, such as during ioctl execution.
132+ */
133+ down_write(&obj->destroy_rwsem);
134+ xa_lock(&ictx->objects);
135+ refcount_dec(&obj->users);
136+ if (!refcount_dec_if_one(&obj->users)) {
137+ xa_unlock(&ictx->objects);
138+ up_write(&obj->destroy_rwsem);
139+ return false;
140+ }
141+ __xa_erase(&ictx->objects, obj->id);
142+ if (ictx->vfio_ioas && &ictx->vfio_ioas->obj == obj)
143+ ictx->vfio_ioas = NULL;
144+ xa_unlock(&ictx->objects);
145+ up_write(&obj->destroy_rwsem);
146+
147+ iommufd_object_ops[obj->type].destroy(obj);
148+ kfree(obj);
149+ return true;
150+}
151+
152+static int iommufd_destroy(struct iommufd_ucmd *ucmd)
153+{
154+ struct iommu_destroy *cmd = ucmd->cmd;
155+ struct iommufd_object *obj;
156+
157+ obj = iommufd_get_object(ucmd->ictx, cmd->id, IOMMUFD_OBJ_ANY);
158+ if (IS_ERR(obj))
159+ return PTR_ERR(obj);
160+ iommufd_ref_to_users(obj);
161+ /* See iommufd_ref_to_users() */
162+ if (!iommufd_object_destroy_user(ucmd->ictx, obj))
163+ return -EBUSY;
164+ return 0;
165+}
166+
167+static int iommufd_fops_open(struct inode *inode, struct file *filp)
168+{
169+ struct iommufd_ctx *ictx;
170+
171+ ictx = kzalloc(sizeof(*ictx), GFP_KERNEL_ACCOUNT);
172+ if (!ictx)
173+ return -ENOMEM;
174+
175+ /*
176+ * For compatibility with VFIO when /dev/vfio/vfio is opened we default
177+ * to the same rlimit accounting as vfio uses.
178+ */
179+ if (IS_ENABLED(CONFIG_IOMMUFD_VFIO_CONTAINER) &&
180+ filp->private_data == &vfio_misc_dev) {
181+ ictx->account_mode = IOPT_PAGES_ACCOUNT_MM;
182+ pr_info_once("IOMMUFD is providing /dev/vfio/vfio, not VFIO.\n");
183+ }
184+
185+ xa_init_flags(&ictx->objects, XA_FLAGS_ALLOC1 | XA_FLAGS_ACCOUNT);
186+ ictx->file = filp;
187+ filp->private_data = ictx;
188+ return 0;
189+}
190+
191+static int iommufd_fops_release(struct inode *inode, struct file *filp)
192+{
193+ struct iommufd_ctx *ictx = filp->private_data;
194+ struct iommufd_object *obj;
195+
196+ /*
197+ * The objects in the xarray form a graph of "users" counts, and we have
198+ * to destroy them in a depth first manner. Leaf objects will reduce the
199+ * users count of interior objects when they are destroyed.
200+ *
201+ * Repeatedly destroying all the "1 users" leaf objects will progress
202+ * until the entire list is destroyed. If this can't progress then there
203+ * is some bug related to object refcounting.
204+ */
205+ while (!xa_empty(&ictx->objects)) {
206+ unsigned int destroyed = 0;
207+ unsigned long index;
208+
209+ xa_for_each(&ictx->objects, index, obj) {
210+ if (!refcount_dec_if_one(&obj->users))
211+ continue;
212+ destroyed++;
213+ xa_erase(&ictx->objects, index);
214+ iommufd_object_ops[obj->type].destroy(obj);
215+ kfree(obj);
216+ }
217+ /* Bug related to users refcount */
218+ if (WARN_ON(!destroyed))
219+ break;
220+ }
221+ kfree(ictx);
222+ return 0;
223+}
224+
225+static int iommufd_option(struct iommufd_ucmd *ucmd)
226+{
227+ struct iommu_option *cmd = ucmd->cmd;
228+ int rc;
229+
230+ if (cmd->__reserved)
231+ return -EOPNOTSUPP;
232+
233+ switch (cmd->option_id) {
234+ case IOMMU_OPTION_RLIMIT_MODE:
235+ rc = iommufd_option_rlimit_mode(cmd, ucmd->ictx);
236+ break;
237+ case IOMMU_OPTION_HUGE_PAGES:
238+ rc = iommufd_ioas_option(ucmd);
239+ break;
240+ default:
241+ return -EOPNOTSUPP;
242+ }
243+ if (rc)
244+ return rc;
245+ if (copy_to_user(&((struct iommu_option __user *)ucmd->ubuffer)->val64,
246+ &cmd->val64, sizeof(cmd->val64)))
247+ return -EFAULT;
248+ return 0;
249+}
250+
251+union ucmd_buffer {
252+ struct iommu_destroy destroy;
253+ struct iommu_ioas_alloc alloc;
254+ struct iommu_ioas_allow_iovas allow_iovas;
255+ struct iommu_ioas_iova_ranges iova_ranges;
256+ struct iommu_ioas_map map;
257+ struct iommu_ioas_unmap unmap;
258+#ifdef CONFIG_IOMMUFD_TEST
259+ struct iommu_test_cmd test;
260+#endif
261+};
262+
263+struct iommufd_ioctl_op {
264+ unsigned int size;
265+ unsigned int min_size;
266+ unsigned int ioctl_num;
267+ int (*execute)(struct iommufd_ucmd *ucmd);
268+};
269+
270+#define IOCTL_OP(_ioctl, _fn, _struct, _last) \
271+ [_IOC_NR(_ioctl) - IOMMUFD_CMD_BASE] = { \
272+ .size = sizeof(_struct) + \
273+ BUILD_BUG_ON_ZERO(sizeof(union ucmd_buffer) < \
274+ sizeof(_struct)), \
275+ .min_size = offsetofend(_struct, _last), \
276+ .ioctl_num = _ioctl, \
277+ .execute = _fn, \
278+ }
279+static const struct iommufd_ioctl_op iommufd_ioctl_ops[] = {
280+ IOCTL_OP(IOMMU_DESTROY, iommufd_destroy, struct iommu_destroy, id),
281+ IOCTL_OP(IOMMU_IOAS_ALLOC, iommufd_ioas_alloc_ioctl,
282+ struct iommu_ioas_alloc, out_ioas_id),
283+ IOCTL_OP(IOMMU_IOAS_ALLOW_IOVAS, iommufd_ioas_allow_iovas,
284+ struct iommu_ioas_allow_iovas, allowed_iovas),
285+ IOCTL_OP(IOMMU_IOAS_COPY, iommufd_ioas_copy, struct iommu_ioas_copy,
286+ src_iova),
287+ IOCTL_OP(IOMMU_IOAS_IOVA_RANGES, iommufd_ioas_iova_ranges,
288+ struct iommu_ioas_iova_ranges, out_iova_alignment),
289+ IOCTL_OP(IOMMU_IOAS_MAP, iommufd_ioas_map, struct iommu_ioas_map,
290+ iova),
291+ IOCTL_OP(IOMMU_IOAS_UNMAP, iommufd_ioas_unmap, struct iommu_ioas_unmap,
292+ length),
293+ IOCTL_OP(IOMMU_OPTION, iommufd_option, struct iommu_option,
294+ val64),
295+ IOCTL_OP(IOMMU_VFIO_IOAS, iommufd_vfio_ioas, struct iommu_vfio_ioas,
296+ __reserved),
297+#ifdef CONFIG_IOMMUFD_TEST
298+ IOCTL_OP(IOMMU_TEST_CMD, iommufd_test, struct iommu_test_cmd, last),
299+#endif
300+};
301+
302+static long iommufd_fops_ioctl(struct file *filp, unsigned int cmd,
303+ unsigned long arg)
304+{
305+ struct iommufd_ctx *ictx = filp->private_data;
306+ const struct iommufd_ioctl_op *op;
307+ struct iommufd_ucmd ucmd = {};
308+ union ucmd_buffer buf;
309+ unsigned int nr;
310+ int ret;
311+
312+ nr = _IOC_NR(cmd);
313+ if (nr < IOMMUFD_CMD_BASE ||
314+ (nr - IOMMUFD_CMD_BASE) >= ARRAY_SIZE(iommufd_ioctl_ops))
315+ return iommufd_vfio_ioctl(ictx, cmd, arg);
316+
317+ ucmd.ictx = ictx;
318+ ucmd.ubuffer = (void __user *)arg;
319+ ret = get_user(ucmd.user_size, (u32 __user *)ucmd.ubuffer);
320+ if (ret)
321+ return ret;
322+
323+ op = &iommufd_ioctl_ops[nr - IOMMUFD_CMD_BASE];
324+ if (op->ioctl_num != cmd)
325+ return -ENOIOCTLCMD;
326+ if (ucmd.user_size < op->min_size)
327+ return -EINVAL;
328+
329+ ucmd.cmd = &buf;
330+ ret = copy_struct_from_user(ucmd.cmd, op->size, ucmd.ubuffer,
331+ ucmd.user_size);
332+ if (ret)
333+ return ret;
334+ ret = op->execute(&ucmd);
335+ return ret;
336+}
337+
338+static const struct file_operations iommufd_fops = {
339+ .owner = THIS_MODULE,
340+ .open = iommufd_fops_open,
341+ .release = iommufd_fops_release,
342+ .unlocked_ioctl = iommufd_fops_ioctl,
343+};
344+
345+/**
346+ * iommufd_ctx_get - Get a context reference
347+ * @ictx: Context to get
348+ *
349+ * The caller must already hold a valid reference to ictx.
350+ */
351+void iommufd_ctx_get(struct iommufd_ctx *ictx)
352+{
353+ get_file(ictx->file);
354+}
355+EXPORT_SYMBOL_NS_GPL(iommufd_ctx_get, IOMMUFD);
356+
357+/**
358+ * iommufd_ctx_from_file - Acquires a reference to the iommufd context
359+ * @file: File to obtain the reference from
360+ *
361+ * Returns a pointer to the iommufd_ctx, otherwise ERR_PTR. The struct file
362+ * remains owned by the caller and the caller must still do fput. On success
363+ * the caller is responsible to call iommufd_ctx_put().
364+ */
365+struct iommufd_ctx *iommufd_ctx_from_file(struct file *file)
366+{
367+ struct iommufd_ctx *ictx;
368+
369+ if (file->f_op != &iommufd_fops)
370+ return ERR_PTR(-EBADFD);
371+ ictx = file->private_data;
372+ iommufd_ctx_get(ictx);
373+ return ictx;
374+}
375+EXPORT_SYMBOL_NS_GPL(iommufd_ctx_from_file, IOMMUFD);
376+
377+/**
378+ * iommufd_ctx_put - Put back a reference
379+ * @ictx: Context to put back
380+ */
381+void iommufd_ctx_put(struct iommufd_ctx *ictx)
382+{
383+ fput(ictx->file);
384+}
385+EXPORT_SYMBOL_NS_GPL(iommufd_ctx_put, IOMMUFD);
386+
387+static const struct iommufd_object_ops iommufd_object_ops[] = {
388+ [IOMMUFD_OBJ_ACCESS] = {
389+ .destroy = iommufd_access_destroy_object,
390+ },
391+ [IOMMUFD_OBJ_DEVICE] = {
392+ .destroy = iommufd_device_destroy,
393+ },
394+ [IOMMUFD_OBJ_IOAS] = {
395+ .destroy = iommufd_ioas_destroy,
396+ },
397+ [IOMMUFD_OBJ_HW_PAGETABLE] = {
398+ .destroy = iommufd_hw_pagetable_destroy,
399+ },
400+#ifdef CONFIG_IOMMUFD_TEST
401+ [IOMMUFD_OBJ_SELFTEST] = {
402+ .destroy = iommufd_selftest_destroy,
403+ },
404+#endif
405+};
406+
407+static struct miscdevice iommu_misc_dev = {
408+ .minor = MISC_DYNAMIC_MINOR,
409+ .name = "iommu",
410+ .fops = &iommufd_fops,
411+ .nodename = "iommu",
412+ .mode = 0660,
413+};
414+
415+
416+static struct miscdevice vfio_misc_dev = {
417+ .minor = VFIO_MINOR,
418+ .name = "vfio",
419+ .fops = &iommufd_fops,
420+ .nodename = "vfio/vfio",
421+ .mode = 0666,
422+};
423+
424+static int __init iommufd_init(void)
425+{
426+ int ret;
427+
428+ ret = misc_register(&iommu_misc_dev);
429+ if (ret)
430+ return ret;
431+
432+ if (IS_ENABLED(CONFIG_IOMMUFD_VFIO_CONTAINER)) {
433+ ret = misc_register(&vfio_misc_dev);
434+ if (ret)
435+ goto err_misc;
436+ }
437+ iommufd_test_init();
438+ return 0;
439+err_misc:
440+ misc_deregister(&iommu_misc_dev);
441+ return ret;
442+}
443+
444+static void __exit iommufd_exit(void)
445+{
446+ iommufd_test_exit();
447+ if (IS_ENABLED(CONFIG_IOMMUFD_VFIO_CONTAINER))
448+ misc_deregister(&vfio_misc_dev);
449+ misc_deregister(&iommu_misc_dev);
450+}
451+
452+module_init(iommufd_init);
453+module_exit(iommufd_exit);
454+
455+#if IS_ENABLED(CONFIG_IOMMUFD_VFIO_CONTAINER)
456+MODULE_ALIAS_MISCDEV(VFIO_MINOR);
457+MODULE_ALIAS("devname:vfio/vfio");
458+#endif
459+MODULE_DESCRIPTION("I/O Address Space Management for passthrough devices");
460+MODULE_LICENSE("GPL");
drivers/iommu/iommufd/pages.c+1977−0
@@ -0,0 +1,1977 @@
1+// SPDX-License-Identifier: GPL-2.0
2+/* Copyright (c) 2021-2022, NVIDIA CORPORATION & AFFILIATES.
3+ *
4+ * The iopt_pages is the center of the storage and motion of PFNs. Each
5+ * iopt_pages represents a logical linear array of full PFNs. The array is 0
6+ * based and has npages in it. Accessors use 'index' to refer to the entry in
7+ * this logical array, regardless of its storage location.
8+ *
9+ * PFNs are stored in a tiered scheme:
10+ * 1) iopt_pages::pinned_pfns xarray
11+ * 2) An iommu_domain
12+ * 3) The origin of the PFNs, i.e. the userspace pointer
13+ *
14+ * PFN have to be copied between all combinations of tiers, depending on the
15+ * configuration.
16+ *
17+ * When a PFN is taken out of the userspace pointer it is pinned exactly once.
18+ * The storage locations of the PFN's index are tracked in the two interval
19+ * trees. If no interval includes the index then it is not pinned.
20+ *
21+ * If access_itree includes the PFN's index then an in-kernel access has
22+ * requested the page. The PFN is stored in the xarray so other requestors can
23+ * continue to find it.
24+ *
25+ * If the domains_itree includes the PFN's index then an iommu_domain is storing
26+ * the PFN and it can be read back using iommu_iova_to_phys(). To avoid
27+ * duplicating storage the xarray is not used if only iommu_domains are using
28+ * the PFN's index.
29+ *
30+ * As a general principle this is designed so that destroy never fails. This
31+ * means removing an iommu_domain or releasing a in-kernel access will not fail
32+ * due to insufficient memory. In practice this means some cases have to hold
33+ * PFNs in the xarray even though they are also being stored in an iommu_domain.
34+ *
35+ * While the iopt_pages can use an iommu_domain as storage, it does not have an
36+ * IOVA itself. Instead the iopt_area represents a range of IOVA and uses the
37+ * iopt_pages as the PFN provider. Multiple iopt_areas can share the iopt_pages
38+ * and reference their own slice of the PFN array, with sub page granularity.
39+ *
40+ * In this file the term 'last' indicates an inclusive and closed interval, eg
41+ * [0,0] refers to a single PFN. 'end' means an open range, eg [0,0) refers to
42+ * no PFNs.
43+ *
44+ * Be cautious of overflow. An IOVA can go all the way up to U64_MAX, so
45+ * last_iova + 1 can overflow. An iopt_pages index will always be much less than
46+ * ULONG_MAX so last_index + 1 cannot overflow.
47+ */
48+#include <linux/overflow.h>
49+#include <linux/slab.h>
50+#include <linux/iommu.h>
51+#include <linux/sched/mm.h>
52+#include <linux/highmem.h>
53+#include <linux/kthread.h>
54+#include <linux/iommufd.h>
55+
56+#include "io_pagetable.h"
57+#include "double_span.h"
58+
59+#ifndef CONFIG_IOMMUFD_TEST
60+#define TEMP_MEMORY_LIMIT 65536
61+#else
62+#define TEMP_MEMORY_LIMIT iommufd_test_memory_limit
63+#endif
64+#define BATCH_BACKUP_SIZE 32
65+
66+/*
67+ * More memory makes pin_user_pages() and the batching more efficient, but as
68+ * this is only a performance optimization don't try too hard to get it. A 64k
69+ * allocation can hold about 26M of 4k pages and 13G of 2M pages in an
70+ * pfn_batch. Various destroy paths cannot fail and provide a small amount of
71+ * stack memory as a backup contingency. If backup_len is given this cannot
72+ * fail.
73+ */
74+static void *temp_kmalloc(size_t *size, void *backup, size_t backup_len)
75+{
76+ void *res;
77+
78+ if (WARN_ON(*size == 0))
79+ return NULL;
80+
81+ if (*size < backup_len)
82+ return backup;
83+
84+ if (!backup && iommufd_should_fail())
85+ return NULL;
86+
87+ *size = min_t(size_t, *size, TEMP_MEMORY_LIMIT);
88+ res = kmalloc(*size, GFP_KERNEL | __GFP_NOWARN | __GFP_NORETRY);
89+ if (res)
90+ return res;
91+ *size = PAGE_SIZE;
92+ if (backup_len) {
93+ res = kmalloc(*size, GFP_KERNEL | __GFP_NOWARN | __GFP_NORETRY);
94+ if (res)
95+ return res;
96+ *size = backup_len;
97+ return backup;
98+ }
99+ return kmalloc(*size, GFP_KERNEL);
100+}
101+
102+void interval_tree_double_span_iter_update(
103+ struct interval_tree_double_span_iter *iter)
104+{
105+ unsigned long last_hole = ULONG_MAX;
106+ unsigned int i;
107+
108+ for (i = 0; i != ARRAY_SIZE(iter->spans); i++) {
109+ if (interval_tree_span_iter_done(&iter->spans[i])) {
110+ iter->is_used = -1;
111+ return;
112+ }
113+
114+ if (iter->spans[i].is_hole) {
115+ last_hole = min(last_hole, iter->spans[i].last_hole);
116+ continue;
117+ }
118+
119+ iter->is_used = i + 1;
120+ iter->start_used = iter->spans[i].start_used;
121+ iter->last_used = min(iter->spans[i].last_used, last_hole);
122+ return;
123+ }
124+
125+ iter->is_used = 0;
126+ iter->start_hole = iter->spans[0].start_hole;
127+ iter->last_hole =
128+ min(iter->spans[0].last_hole, iter->spans[1].last_hole);
129+}
130+
131+void interval_tree_double_span_iter_first(
132+ struct interval_tree_double_span_iter *iter,
133+ struct rb_root_cached *itree1, struct rb_root_cached *itree2,
134+ unsigned long first_index, unsigned long last_index)
135+{
136+ unsigned int i;
137+
138+ iter->itrees[0] = itree1;
139+ iter->itrees[1] = itree2;
140+ for (i = 0; i != ARRAY_SIZE(iter->spans); i++)
141+ interval_tree_span_iter_first(&iter->spans[i], iter->itrees[i],
142+ first_index, last_index);
143+ interval_tree_double_span_iter_update(iter);
144+}
145+
146+void interval_tree_double_span_iter_next(
147+ struct interval_tree_double_span_iter *iter)
148+{
149+ unsigned int i;
150+
151+ if (iter->is_used == -1 ||
152+ iter->last_hole == iter->spans[0].last_index) {
153+ iter->is_used = -1;
154+ return;
155+ }
156+
157+ for (i = 0; i != ARRAY_SIZE(iter->spans); i++)
158+ interval_tree_span_iter_advance(
159+ &iter->spans[i], iter->itrees[i], iter->last_hole + 1);
160+ interval_tree_double_span_iter_update(iter);
161+}
162+
163+static void iopt_pages_add_npinned(struct iopt_pages *pages, size_t npages)
164+{
165+ int rc;
166+
167+ rc = check_add_overflow(pages->npinned, npages, &pages->npinned);
168+ if (IS_ENABLED(CONFIG_IOMMUFD_TEST))
169+ WARN_ON(rc || pages->npinned > pages->npages);
170+}
171+
172+static void iopt_pages_sub_npinned(struct iopt_pages *pages, size_t npages)
173+{
174+ int rc;
175+
176+ rc = check_sub_overflow(pages->npinned, npages, &pages->npinned);
177+ if (IS_ENABLED(CONFIG_IOMMUFD_TEST))
178+ WARN_ON(rc || pages->npinned > pages->npages);
179+}
180+
181+static void iopt_pages_err_unpin(struct iopt_pages *pages,
182+ unsigned long start_index,
183+ unsigned long last_index,
184+ struct page **page_list)
185+{
186+ unsigned long npages = last_index - start_index + 1;
187+
188+ unpin_user_pages(page_list, npages);
189+ iopt_pages_sub_npinned(pages, npages);
190+}
191+
192+/*
193+ * index is the number of PAGE_SIZE units from the start of the area's
194+ * iopt_pages. If the iova is sub page-size then the area has an iova that
195+ * covers a portion of the first and last pages in the range.
196+ */
197+static unsigned long iopt_area_index_to_iova(struct iopt_area *area,
198+ unsigned long index)
199+{
200+ if (IS_ENABLED(CONFIG_IOMMUFD_TEST))
201+ WARN_ON(index < iopt_area_index(area) ||
202+ index > iopt_area_last_index(area));
203+ index -= iopt_area_index(area);
204+ if (index == 0)
205+ return iopt_area_iova(area);
206+ return iopt_area_iova(area) - area->page_offset + index * PAGE_SIZE;
207+}
208+
209+static unsigned long iopt_area_index_to_iova_last(struct iopt_area *area,
210+ unsigned long index)
211+{
212+ if (IS_ENABLED(CONFIG_IOMMUFD_TEST))
213+ WARN_ON(index < iopt_area_index(area) ||
214+ index > iopt_area_last_index(area));
215+ if (index == iopt_area_last_index(area))
216+ return iopt_area_last_iova(area);
217+ return iopt_area_iova(area) - area->page_offset +
218+ (index - iopt_area_index(area) + 1) * PAGE_SIZE - 1;
219+}
220+
221+static void iommu_unmap_nofail(struct iommu_domain *domain, unsigned long iova,
222+ size_t size)
223+{
224+ size_t ret;
225+
226+ ret = iommu_unmap(domain, iova, size);
227+ /*
228+ * It is a logic error in this code or a driver bug if the IOMMU unmaps
229+ * something other than exactly as requested. This implies that the
230+ * iommu driver may not fail unmap for reasons beyond bad agruments.
231+ * Particularly, the iommu driver may not do a memory allocation on the
232+ * unmap path.
233+ */
234+ WARN_ON(ret != size);
235+}
236+
237+static void iopt_area_unmap_domain_range(struct iopt_area *area,
238+ struct iommu_domain *domain,
239+ unsigned long start_index,
240+ unsigned long last_index)
241+{
242+ unsigned long start_iova = iopt_area_index_to_iova(area, start_index);
243+
244+ iommu_unmap_nofail(domain, start_iova,
245+ iopt_area_index_to_iova_last(area, last_index) -
246+ start_iova + 1);
247+}
248+
249+static struct iopt_area *iopt_pages_find_domain_area(struct iopt_pages *pages,
250+ unsigned long index)
251+{
252+ struct interval_tree_node *node;
253+
254+ node = interval_tree_iter_first(&pages->domains_itree, index, index);
255+ if (!node)
256+ return NULL;
257+ return container_of(node, struct iopt_area, pages_node);
258+}
259+
260+/*
261+ * A simple datastructure to hold a vector of PFNs, optimized for contiguous
262+ * PFNs. This is used as a temporary holding memory for shuttling pfns from one
263+ * place to another. Generally everything is made more efficient if operations
264+ * work on the largest possible grouping of pfns. eg fewer lock/unlock cycles,
265+ * better cache locality, etc
266+ */
267+struct pfn_batch {
268+ unsigned long *pfns;
269+ u32 *npfns;
270+ unsigned int array_size;
271+ unsigned int end;
272+ unsigned int total_pfns;
273+};
274+
275+static void batch_clear(struct pfn_batch *batch)
276+{
277+ batch->total_pfns = 0;
278+ batch->end = 0;
279+ batch->pfns[0] = 0;
280+ batch->npfns[0] = 0;
281+}
282+
283+/*
284+ * Carry means we carry a portion of the final hugepage over to the front of the
285+ * batch
286+ */
287+static void batch_clear_carry(struct pfn_batch *batch, unsigned int keep_pfns)
288+{
289+ if (!keep_pfns)
290+ return batch_clear(batch);
291+
292+ if (IS_ENABLED(CONFIG_IOMMUFD_TEST))
293+ WARN_ON(!batch->end ||
294+ batch->npfns[batch->end - 1] < keep_pfns);
295+
296+ batch->total_pfns = keep_pfns;
297+ batch->npfns[0] = keep_pfns;
298+ batch->pfns[0] = batch->pfns[batch->end - 1] +
299+ (batch->npfns[batch->end - 1] - keep_pfns);
300+ batch->end = 0;
301+}
302+
303+static void batch_skip_carry(struct pfn_batch *batch, unsigned int skip_pfns)
304+{
305+ if (!batch->total_pfns)
306+ return;
307+ if (IS_ENABLED(CONFIG_IOMMUFD_TEST))
308+ WARN_ON(batch->total_pfns != batch->npfns[0]);
309+ skip_pfns = min(batch->total_pfns, skip_pfns);
310+ batch->pfns[0] += skip_pfns;
311+ batch->npfns[0] -= skip_pfns;
312+ batch->total_pfns -= skip_pfns;
313+}
314+
315+static int __batch_init(struct pfn_batch *batch, size_t max_pages, void *backup,
316+ size_t backup_len)
317+{
318+ const size_t elmsz = sizeof(*batch->pfns) + sizeof(*batch->npfns);
319+ size_t size = max_pages * elmsz;
320+
321+ batch->pfns = temp_kmalloc(&size, backup, backup_len);
322+ if (!batch->pfns)
323+ return -ENOMEM;
324+ if (IS_ENABLED(CONFIG_IOMMUFD_TEST) && WARN_ON(size < elmsz))
325+ return -EINVAL;
326+ batch->array_size = size / elmsz;
327+ batch->npfns = (u32 *)(batch->pfns + batch->array_size);
328+ batch_clear(batch);
329+ return 0;
330+}
331+
332+static int batch_init(struct pfn_batch *batch, size_t max_pages)
333+{
334+ return __batch_init(batch, max_pages, NULL, 0);
335+}
336+
337+static void batch_init_backup(struct pfn_batch *batch, size_t max_pages,
338+ void *backup, size_t backup_len)
339+{
340+ __batch_init(batch, max_pages, backup, backup_len);
341+}
342+
343+static void batch_destroy(struct pfn_batch *batch, void *backup)
344+{
345+ if (batch->pfns != backup)
346+ kfree(batch->pfns);
347+}
348+
349+/* true if the pfn was added, false otherwise */
350+static bool batch_add_pfn(struct pfn_batch *batch, unsigned long pfn)
351+{
352+ const unsigned int MAX_NPFNS = type_max(typeof(*batch->npfns));
353+
354+ if (batch->end &&
355+ pfn == batch->pfns[batch->end - 1] + batch->npfns[batch->end - 1] &&
356+ batch->npfns[batch->end - 1] != MAX_NPFNS) {
357+ batch->npfns[batch->end - 1]++;
358+ batch->total_pfns++;
359+ return true;
360+ }
361+ if (batch->end == batch->array_size)
362+ return false;
363+ batch->total_pfns++;
364+ batch->pfns[batch->end] = pfn;
365+ batch->npfns[batch->end] = 1;
366+ batch->end++;
367+ return true;
368+}
369+
370+/*
371+ * Fill the batch with pfns from the domain. When the batch is full, or it
372+ * reaches last_index, the function will return. The caller should use
373+ * batch->total_pfns to determine the starting point for the next iteration.
374+ */
375+static void batch_from_domain(struct pfn_batch *batch,
376+ struct iommu_domain *domain,
377+ struct iopt_area *area, unsigned long start_index,
378+ unsigned long last_index)
379+{
380+ unsigned int page_offset = 0;
381+ unsigned long iova;
382+ phys_addr_t phys;
383+
384+ iova = iopt_area_index_to_iova(area, start_index);
385+ if (start_index == iopt_area_index(area))
386+ page_offset = area->page_offset;
387+ while (start_index <= last_index) {
388+ /*
389+ * This is pretty slow, it would be nice to get the page size
390+ * back from the driver, or have the driver directly fill the
391+ * batch.
392+ */
393+ phys = iommu_iova_to_phys(domain, iova) - page_offset;
394+ if (!batch_add_pfn(batch, PHYS_PFN(phys)))
395+ return;
396+ iova += PAGE_SIZE - page_offset;
397+ page_offset = 0;
398+ start_index++;
399+ }
400+}
401+
402+static struct page **raw_pages_from_domain(struct iommu_domain *domain,
403+ struct iopt_area *area,
404+ unsigned long start_index,
405+ unsigned long last_index,
406+ struct page **out_pages)
407+{
408+ unsigned int page_offset = 0;
409+ unsigned long iova;
410+ phys_addr_t phys;
411+
412+ iova = iopt_area_index_to_iova(area, start_index);
413+ if (start_index == iopt_area_index(area))
414+ page_offset = area->page_offset;
415+ while (start_index <= last_index) {
416+ phys = iommu_iova_to_phys(domain, iova) - page_offset;
417+ *(out_pages++) = pfn_to_page(PHYS_PFN(phys));
418+ iova += PAGE_SIZE - page_offset;
419+ page_offset = 0;
420+ start_index++;
421+ }
422+ return out_pages;
423+}
424+
425+/* Continues reading a domain until we reach a discontinuity in the pfns. */
426+static void batch_from_domain_continue(struct pfn_batch *batch,
427+ struct iommu_domain *domain,
428+ struct iopt_area *area,
429+ unsigned long start_index,
430+ unsigned long last_index)
431+{
432+ unsigned int array_size = batch->array_size;
433+
434+ batch->array_size = batch->end;
435+ batch_from_domain(batch, domain, area, start_index, last_index);
436+ batch->array_size = array_size;
437+}
438+
439+/*
440+ * This is part of the VFIO compatibility support for VFIO_TYPE1_IOMMU. That
441+ * mode permits splitting a mapped area up, and then one of the splits is
442+ * unmapped. Doing this normally would cause us to violate our invariant of
443+ * pairing map/unmap. Thus, to support old VFIO compatibility disable support
444+ * for batching consecutive PFNs. All PFNs mapped into the iommu are done in
445+ * PAGE_SIZE units, not larger or smaller.
446+ */
447+static int batch_iommu_map_small(struct iommu_domain *domain,
448+ unsigned long iova, phys_addr_t paddr,
449+ size_t size, int prot)
450+{
451+ unsigned long start_iova = iova;
452+ int rc;
453+
454+ if (IS_ENABLED(CONFIG_IOMMUFD_TEST))
455+ WARN_ON(paddr % PAGE_SIZE || iova % PAGE_SIZE ||
456+ size % PAGE_SIZE);
457+
458+ while (size) {
459+ rc = iommu_map(domain, iova, paddr, PAGE_SIZE, prot);
460+ if (rc)
461+ goto err_unmap;
462+ iova += PAGE_SIZE;
463+ paddr += PAGE_SIZE;
464+ size -= PAGE_SIZE;
465+ }
466+ return 0;
467+
468+err_unmap:
469+ if (start_iova != iova)
470+ iommu_unmap_nofail(domain, start_iova, iova - start_iova);
471+ return rc;
472+}
473+
474+static int batch_to_domain(struct pfn_batch *batch, struct iommu_domain *domain,
475+ struct iopt_area *area, unsigned long start_index)
476+{
477+ bool disable_large_pages = area->iopt->disable_large_pages;
478+ unsigned long last_iova = iopt_area_last_iova(area);
479+ unsigned int page_offset = 0;
480+ unsigned long start_iova;
481+ unsigned long next_iova;
482+ unsigned int cur = 0;
483+ unsigned long iova;
484+ int rc;
485+
486+ /* The first index might be a partial page */
487+ if (start_index == iopt_area_index(area))
488+ page_offset = area->page_offset;
489+ next_iova = iova = start_iova =
490+ iopt_area_index_to_iova(area, start_index);
491+ while (cur < batch->end) {
492+ next_iova = min(last_iova + 1,
493+ next_iova + batch->npfns[cur] * PAGE_SIZE -
494+ page_offset);
495+ if (disable_large_pages)
496+ rc = batch_iommu_map_small(
497+ domain, iova,
498+ PFN_PHYS(batch->pfns[cur]) + page_offset,
499+ next_iova - iova, area->iommu_prot);
500+ else
501+ rc = iommu_map(domain, iova,
502+ PFN_PHYS(batch->pfns[cur]) + page_offset,
503+ next_iova - iova, area->iommu_prot);
504+ if (rc)
505+ goto err_unmap;
506+ iova = next_iova;
507+ page_offset = 0;
508+ cur++;
509+ }
510+ return 0;
511+err_unmap:
512+ if (start_iova != iova)
513+ iommu_unmap_nofail(domain, start_iova, iova - start_iova);
514+ return rc;
515+}
516+
517+static void batch_from_xarray(struct pfn_batch *batch, struct xarray *xa,
518+ unsigned long start_index,
519+ unsigned long last_index)
520+{
521+ XA_STATE(xas, xa, start_index);
522+ void *entry;
523+
524+ rcu_read_lock();
525+ while (true) {
526+ entry = xas_next(&xas);
527+ if (xas_retry(&xas, entry))
528+ continue;
529+ WARN_ON(!xa_is_value(entry));
530+ if (!batch_add_pfn(batch, xa_to_value(entry)) ||
531+ start_index == last_index)
532+ break;
533+ start_index++;
534+ }
535+ rcu_read_unlock();
536+}
537+
538+static void batch_from_xarray_clear(struct pfn_batch *batch, struct xarray *xa,
539+ unsigned long start_index,
540+ unsigned long last_index)
541+{
542+ XA_STATE(xas, xa, start_index);
543+ void *entry;
544+
545+ xas_lock(&xas);
546+ while (true) {
547+ entry = xas_next(&xas);
548+ if (xas_retry(&xas, entry))
549+ continue;
550+ WARN_ON(!xa_is_value(entry));
551+ if (!batch_add_pfn(batch, xa_to_value(entry)))
552+ break;
553+ xas_store(&xas, NULL);
554+ if (start_index == last_index)
555+ break;
556+ start_index++;
557+ }
558+ xas_unlock(&xas);
559+}
560+
561+static void clear_xarray(struct xarray *xa, unsigned long start_index,
562+ unsigned long last_index)
563+{
564+ XA_STATE(xas, xa, start_index);
565+ void *entry;
566+
567+ xas_lock(&xas);
568+ xas_for_each(&xas, entry, last_index)
569+ xas_store(&xas, NULL);
570+ xas_unlock(&xas);
571+}
572+
573+static int pages_to_xarray(struct xarray *xa, unsigned long start_index,
574+ unsigned long last_index, struct page **pages)
575+{
576+ struct page **end_pages = pages + (last_index - start_index) + 1;
577+ struct page **half_pages = pages + (end_pages - pages) / 2;
578+ XA_STATE(xas, xa, start_index);
579+
580+ do {
581+ void *old;
582+
583+ xas_lock(&xas);
584+ while (pages != end_pages) {
585+ /* xarray does not participate in fault injection */
586+ if (pages == half_pages && iommufd_should_fail()) {
587+ xas_set_err(&xas, -EINVAL);
588+ xas_unlock(&xas);
589+ /* aka xas_destroy() */
590+ xas_nomem(&xas, GFP_KERNEL);
591+ goto err_clear;
592+ }
593+
594+ old = xas_store(&xas, xa_mk_value(page_to_pfn(*pages)));
595+ if (xas_error(&xas))
596+ break;
597+ WARN_ON(old);
598+ pages++;
599+ xas_next(&xas);
600+ }
601+ xas_unlock(&xas);
602+ } while (xas_nomem(&xas, GFP_KERNEL));
603+
604+err_clear:
605+ if (xas_error(&xas)) {
606+ if (xas.xa_index != start_index)
607+ clear_xarray(xa, start_index, xas.xa_index - 1);
608+ return xas_error(&xas);
609+ }
610+ return 0;
611+}
612+
613+static void batch_from_pages(struct pfn_batch *batch, struct page **pages,
614+ size_t npages)
615+{
616+ struct page **end = pages + npages;
617+
618+ for (; pages != end; pages++)
619+ if (!batch_add_pfn(batch, page_to_pfn(*pages)))
620+ break;
621+}
622+
623+static void batch_unpin(struct pfn_batch *batch, struct iopt_pages *pages,
624+ unsigned int first_page_off, size_t npages)
625+{
626+ unsigned int cur = 0;
627+
628+ while (first_page_off) {
629+ if (batch->npfns[cur] > first_page_off)
630+ break;
631+ first_page_off -= batch->npfns[cur];
632+ cur++;
633+ }
634+
635+ while (npages) {
636+ size_t to_unpin = min_t(size_t, npages,
637+ batch->npfns[cur] - first_page_off);
638+
639+ unpin_user_page_range_dirty_lock(
640+ pfn_to_page(batch->pfns[cur] + first_page_off),
641+ to_unpin, pages->writable);
642+ iopt_pages_sub_npinned(pages, to_unpin);
643+ cur++;
644+ first_page_off = 0;
645+ npages -= to_unpin;
646+ }
647+}
648+
649+static void copy_data_page(struct page *page, void *data, unsigned long offset,
650+ size_t length, unsigned int flags)
651+{
652+ void *mem;
653+
654+ mem = kmap_local_page(page);
655+ if (flags & IOMMUFD_ACCESS_RW_WRITE) {
656+ memcpy(mem + offset, data, length);
657+ set_page_dirty_lock(page);
658+ } else {
659+ memcpy(data, mem + offset, length);
660+ }
661+ kunmap_local(mem);
662+}
663+
664+static unsigned long batch_rw(struct pfn_batch *batch, void *data,
665+ unsigned long offset, unsigned long length,
666+ unsigned int flags)
667+{
668+ unsigned long copied = 0;
669+ unsigned int npage = 0;
670+ unsigned int cur = 0;
671+
672+ while (cur < batch->end) {
673+ unsigned long bytes = min(length, PAGE_SIZE - offset);
674+
675+ copy_data_page(pfn_to_page(batch->pfns[cur] + npage), data,
676+ offset, bytes, flags);
677+ offset = 0;
678+ length -= bytes;
679+ data += bytes;
680+ copied += bytes;
681+ npage++;
682+ if (npage == batch->npfns[cur]) {
683+ npage = 0;
684+ cur++;
685+ }
686+ if (!length)
687+ break;
688+ }
689+ return copied;
690+}
691+
692+/* pfn_reader_user is just the pin_user_pages() path */
693+struct pfn_reader_user {
694+ struct page **upages;
695+ size_t upages_len;
696+ unsigned long upages_start;
697+ unsigned long upages_end;
698+ unsigned int gup_flags;
699+ /*
700+ * 1 means mmget() and mmap_read_lock(), 0 means only mmget(), -1 is
701+ * neither
702+ */
703+ int locked;
704+};
705+
706+static void pfn_reader_user_init(struct pfn_reader_user *user,
707+ struct iopt_pages *pages)
708+{
709+ user->upages = NULL;
710+ user->upages_start = 0;
711+ user->upages_end = 0;
712+ user->locked = -1;
713+
714+ user->gup_flags = FOLL_LONGTERM;
715+ if (pages->writable)
716+ user->gup_flags |= FOLL_WRITE;
717+}
718+
719+static void pfn_reader_user_destroy(struct pfn_reader_user *user,
720+ struct iopt_pages *pages)
721+{
722+ if (user->locked != -1) {
723+ if (user->locked)
724+ mmap_read_unlock(pages->source_mm);
725+ if (pages->source_mm != current->mm)
726+ mmput(pages->source_mm);
727+ user->locked = -1;
728+ }
729+
730+ kfree(user->upages);
731+ user->upages = NULL;
732+}
733+
734+static int pfn_reader_user_pin(struct pfn_reader_user *user,
735+ struct iopt_pages *pages,
736+ unsigned long start_index,
737+ unsigned long last_index)
738+{
739+ bool remote_mm = pages->source_mm != current->mm;
740+ unsigned long npages;
741+ uintptr_t uptr;
742+ long rc;
743+
744+ if (IS_ENABLED(CONFIG_IOMMUFD_TEST) &&
745+ WARN_ON(last_index < start_index))
746+ return -EINVAL;
747+
748+ if (!user->upages) {
749+ /* All undone in pfn_reader_destroy() */
750+ user->upages_len =
751+ (last_index - start_index + 1) * sizeof(*user->upages);
752+ user->upages = temp_kmalloc(&user->upages_len, NULL, 0);
753+ if (!user->upages)
754+ return -ENOMEM;
755+ }
756+
757+ if (user->locked == -1) {
758+ /*
759+ * The majority of usages will run the map task within the mm
760+ * providing the pages, so we can optimize into
761+ * get_user_pages_fast()
762+ */
763+ if (remote_mm) {
764+ if (!mmget_not_zero(pages->source_mm))
765+ return -EFAULT;
766+ }
767+ user->locked = 0;
768+ }
769+
770+ npages = min_t(unsigned long, last_index - start_index + 1,
771+ user->upages_len / sizeof(*user->upages));
772+
773+
774+ if (iommufd_should_fail())
775+ return -EFAULT;
776+
777+ uptr = (uintptr_t)(pages->uptr + start_index * PAGE_SIZE);
778+ if (!remote_mm)
779+ rc = pin_user_pages_fast(uptr, npages, user->gup_flags,
780+ user->upages);
781+ else {
782+ if (!user->locked) {
783+ mmap_read_lock(pages->source_mm);
784+ user->locked = 1;
785+ }
786+ rc = pin_user_pages_remote(pages->source_mm, uptr, npages,
787+ user->gup_flags, user->upages, NULL,
788+ &user->locked);
789+ }
790+ if (rc <= 0) {
791+ if (WARN_ON(!rc))
792+ return -EFAULT;
793+ return rc;
794+ }
795+ iopt_pages_add_npinned(pages, rc);
796+ user->upages_start = start_index;
797+ user->upages_end = start_index + rc;
798+ return 0;
799+}
800+
801+/* This is the "modern" and faster accounting method used by io_uring */
802+static int incr_user_locked_vm(struct iopt_pages *pages, unsigned long npages)
803+{
804+ unsigned long lock_limit;
805+ unsigned long cur_pages;
806+ unsigned long new_pages;
807+
808+ lock_limit = task_rlimit(pages->source_task, RLIMIT_MEMLOCK) >>
809+ PAGE_SHIFT;
810+ do {
811+ cur_pages = atomic_long_read(&pages->source_user->locked_vm);
812+ new_pages = cur_pages + npages;
813+ if (new_pages > lock_limit)
814+ return -ENOMEM;
815+ } while (atomic_long_cmpxchg(&pages->source_user->locked_vm, cur_pages,
816+ new_pages) != cur_pages);
817+ return 0;
818+}
819+
820+static void decr_user_locked_vm(struct iopt_pages *pages, unsigned long npages)
821+{
822+ if (WARN_ON(atomic_long_read(&pages->source_user->locked_vm) < npages))
823+ return;
824+ atomic_long_sub(npages, &pages->source_user->locked_vm);
825+}
826+
827+/* This is the accounting method used for compatibility with VFIO */
828+static int update_mm_locked_vm(struct iopt_pages *pages, unsigned long npages,
829+ bool inc, struct pfn_reader_user *user)
830+{
831+ bool do_put = false;
832+ int rc;
833+
834+ if (user && user->locked) {
835+ mmap_read_unlock(pages->source_mm);
836+ user->locked = 0;
837+ /* If we had the lock then we also have a get */
838+ } else if ((!user || !user->upages) &&
839+ pages->source_mm != current->mm) {
840+ if (!mmget_not_zero(pages->source_mm))
841+ return -EINVAL;
842+ do_put = true;
843+ }
844+
845+ mmap_write_lock(pages->source_mm);
846+ rc = __account_locked_vm(pages->source_mm, npages, inc,
847+ pages->source_task, false);
848+ mmap_write_unlock(pages->source_mm);
849+
850+ if (do_put)
851+ mmput(pages->source_mm);
852+ return rc;
853+}
854+
855+static int do_update_pinned(struct iopt_pages *pages, unsigned long npages,
856+ bool inc, struct pfn_reader_user *user)
857+{
858+ int rc = 0;
859+
860+ switch (pages->account_mode) {
861+ case IOPT_PAGES_ACCOUNT_NONE:
862+ break;
863+ case IOPT_PAGES_ACCOUNT_USER:
864+ if (inc)
865+ rc = incr_user_locked_vm(pages, npages);
866+ else
867+ decr_user_locked_vm(pages, npages);
868+ break;
869+ case IOPT_PAGES_ACCOUNT_MM:
870+ rc = update_mm_locked_vm(pages, npages, inc, user);
871+ break;
872+ }
873+ if (rc)
874+ return rc;
875+
876+ pages->last_npinned = pages->npinned;
877+ if (inc)
878+ atomic64_add(npages, &pages->source_mm->pinned_vm);
879+ else
880+ atomic64_sub(npages, &pages->source_mm->pinned_vm);
881+ return 0;
882+}
883+
884+static void update_unpinned(struct iopt_pages *pages)
885+{
886+ if (WARN_ON(pages->npinned > pages->last_npinned))
887+ return;
888+ if (pages->npinned == pages->last_npinned)
889+ return;
890+ do_update_pinned(pages, pages->last_npinned - pages->npinned, false,
891+ NULL);
892+}
893+
894+/*
895+ * Changes in the number of pages pinned is done after the pages have been read
896+ * and processed. If the user lacked the limit then the error unwind will unpin
897+ * everything that was just pinned. This is because it is expensive to calculate
898+ * how many pages we have already pinned within a range to generate an accurate
899+ * prediction in advance of doing the work to actually pin them.
900+ */
901+static int pfn_reader_user_update_pinned(struct pfn_reader_user *user,
902+ struct iopt_pages *pages)
903+{
904+ unsigned long npages;
905+ bool inc;
906+
907+ lockdep_assert_held(&pages->mutex);
908+
909+ if (pages->npinned == pages->last_npinned)
910+ return 0;
911+
912+ if (pages->npinned < pages->last_npinned) {
913+ npages = pages->last_npinned - pages->npinned;
914+ inc = false;
915+ } else {
916+ if (iommufd_should_fail())
917+ return -ENOMEM;
918+ npages = pages->npinned - pages->last_npinned;
919+ inc = true;
920+ }
921+ return do_update_pinned(pages, npages, inc, user);
922+}
923+
924+/*
925+ * PFNs are stored in three places, in order of preference:
926+ * - The iopt_pages xarray. This is only populated if there is a
927+ * iopt_pages_access
928+ * - The iommu_domain under an area
929+ * - The original PFN source, ie pages->source_mm
930+ *
931+ * This iterator reads the pfns optimizing to load according to the
932+ * above order.
933+ */
934+struct pfn_reader {
935+ struct iopt_pages *pages;
936+ struct interval_tree_double_span_iter span;
937+ struct pfn_batch batch;
938+ unsigned long batch_start_index;
939+ unsigned long batch_end_index;
940+ unsigned long last_index;
941+
942+ struct pfn_reader_user user;
943+};
944+
945+static int pfn_reader_update_pinned(struct pfn_reader *pfns)
946+{
947+ return pfn_reader_user_update_pinned(&pfns->user, pfns->pages);
948+}
949+
950+/*
951+ * The batch can contain a mixture of pages that are still in use and pages that
952+ * need to be unpinned. Unpin only pages that are not held anywhere else.
953+ */
954+static void pfn_reader_unpin(struct pfn_reader *pfns)
955+{
956+ unsigned long last = pfns->batch_end_index - 1;
957+ unsigned long start = pfns->batch_start_index;
958+ struct interval_tree_double_span_iter span;
959+ struct iopt_pages *pages = pfns->pages;
960+
961+ lockdep_assert_held(&pages->mutex);
962+
963+ interval_tree_for_each_double_span(&span, &pages->access_itree,
964+ &pages->domains_itree, start, last) {
965+ if (span.is_used)
966+ continue;
967+
968+ batch_unpin(&pfns->batch, pages, span.start_hole - start,
969+ span.last_hole - span.start_hole + 1);
970+ }
971+}
972+
973+/* Process a single span to load it from the proper storage */
974+static int pfn_reader_fill_span(struct pfn_reader *pfns)
975+{
976+ struct interval_tree_double_span_iter *span = &pfns->span;
977+ unsigned long start_index = pfns->batch_end_index;
978+ struct iopt_area *area;
979+ int rc;
980+
981+ if (IS_ENABLED(CONFIG_IOMMUFD_TEST) &&
982+ WARN_ON(span->last_used < start_index))
983+ return -EINVAL;
984+
985+ if (span->is_used == 1) {
986+ batch_from_xarray(&pfns->batch, &pfns->pages->pinned_pfns,
987+ start_index, span->last_used);
988+ return 0;
989+ }
990+
991+ if (span->is_used == 2) {
992+ /*
993+ * Pull as many pages from the first domain we find in the
994+ * target span. If it is too small then we will be called again
995+ * and we'll find another area.
996+ */
997+ area = iopt_pages_find_domain_area(pfns->pages, start_index);
998+ if (WARN_ON(!area))
999+ return -EINVAL;
1000+
1001+ /* The storage_domain cannot change without the pages mutex */
1002+ batch_from_domain(
1003+ &pfns->batch, area->storage_domain, area, start_index,
1004+ min(iopt_area_last_index(area), span->last_used));
1005+ return 0;
1006+ }
1007+
1008+ if (start_index >= pfns->user.upages_end) {
1009+ rc = pfn_reader_user_pin(&pfns->user, pfns->pages, start_index,
1010+ span->last_hole);
1011+ if (rc)
1012+ return rc;
1013+ }
1014+
1015+ batch_from_pages(&pfns->batch,
1016+ pfns->user.upages +
1017+ (start_index - pfns->user.upages_start),
1018+ pfns->user.upages_end - start_index);
1019+ return 0;
1020+}
1021+
1022+static bool pfn_reader_done(struct pfn_reader *pfns)
1023+{
1024+ return pfns->batch_start_index == pfns->last_index + 1;
1025+}
1026+
1027+static int pfn_reader_next(struct pfn_reader *pfns)
1028+{
1029+ int rc;
1030+
1031+ batch_clear(&pfns->batch);
1032+ pfns->batch_start_index = pfns->batch_end_index;
1033+
1034+ while (pfns->batch_end_index != pfns->last_index + 1) {
1035+ unsigned int npfns = pfns->batch.total_pfns;
1036+
1037+ if (IS_ENABLED(CONFIG_IOMMUFD_TEST) &&
1038+ WARN_ON(interval_tree_double_span_iter_done(&pfns->span)))
1039+ return -EINVAL;
1040+
1041+ rc = pfn_reader_fill_span(pfns);
1042+ if (rc)
1043+ return rc;
1044+
1045+ if (WARN_ON(!pfns->batch.total_pfns))
1046+ return -EINVAL;
1047+
1048+ pfns->batch_end_index =
1049+ pfns->batch_start_index + pfns->batch.total_pfns;
1050+ if (pfns->batch_end_index == pfns->span.last_used + 1)
1051+ interval_tree_double_span_iter_next(&pfns->span);
1052+
1053+ /* Batch is full */
1054+ if (npfns == pfns->batch.total_pfns)
1055+ return 0;
1056+ }
1057+ return 0;
1058+}
1059+
1060+static int pfn_reader_init(struct pfn_reader *pfns, struct iopt_pages *pages,
1061+ unsigned long start_index, unsigned long last_index)
1062+{
1063+ int rc;
1064+
1065+ lockdep_assert_held(&pages->mutex);
1066+
1067+ pfns->pages = pages;
1068+ pfns->batch_start_index = start_index;
1069+ pfns->batch_end_index = start_index;
1070+ pfns->last_index = last_index;
1071+ pfn_reader_user_init(&pfns->user, pages);
1072+ rc = batch_init(&pfns->batch, last_index - start_index + 1);
1073+ if (rc)
1074+ return rc;
1075+ interval_tree_double_span_iter_first(&pfns->span, &pages->access_itree,
1076+ &pages->domains_itree, start_index,
1077+ last_index);
1078+ return 0;
1079+}
1080+
1081+/*
1082+ * There are many assertions regarding the state of pages->npinned vs
1083+ * pages->last_pinned, for instance something like unmapping a domain must only
1084+ * decrement the npinned, and pfn_reader_destroy() must be called only after all
1085+ * the pins are updated. This is fine for success flows, but error flows
1086+ * sometimes need to release the pins held inside the pfn_reader before going on
1087+ * to complete unmapping and releasing pins held in domains.
1088+ */
1089+static void pfn_reader_release_pins(struct pfn_reader *pfns)
1090+{
1091+ struct iopt_pages *pages = pfns->pages;
1092+
1093+ if (pfns->user.upages_end > pfns->batch_end_index) {
1094+ size_t npages = pfns->user.upages_end - pfns->batch_end_index;
1095+
1096+ /* Any pages not transferred to the batch are just unpinned */
1097+ unpin_user_pages(pfns->user.upages + (pfns->batch_end_index -
1098+ pfns->user.upages_start),
1099+ npages);
1100+ iopt_pages_sub_npinned(pages, npages);
1101+ pfns->user.upages_end = pfns->batch_end_index;
1102+ }
1103+ if (pfns->batch_start_index != pfns->batch_end_index) {
1104+ pfn_reader_unpin(pfns);
1105+ pfns->batch_start_index = pfns->batch_end_index;
1106+ }
1107+}
1108+
1109+static void pfn_reader_destroy(struct pfn_reader *pfns)
1110+{
1111+ struct iopt_pages *pages = pfns->pages;
1112+
1113+ pfn_reader_release_pins(pfns);
1114+ pfn_reader_user_destroy(&pfns->user, pfns->pages);
1115+ batch_destroy(&pfns->batch, NULL);
1116+ WARN_ON(pages->last_npinned != pages->npinned);
1117+}
1118+
1119+static int pfn_reader_first(struct pfn_reader *pfns, struct iopt_pages *pages,
1120+ unsigned long start_index, unsigned long last_index)
1121+{
1122+ int rc;
1123+
1124+ if (IS_ENABLED(CONFIG_IOMMUFD_TEST) &&
1125+ WARN_ON(last_index < start_index))
1126+ return -EINVAL;
1127+
1128+ rc = pfn_reader_init(pfns, pages, start_index, last_index);
1129+ if (rc)
1130+ return rc;
1131+ rc = pfn_reader_next(pfns);
1132+ if (rc) {
1133+ pfn_reader_destroy(pfns);
1134+ return rc;
1135+ }
1136+ return 0;
1137+}
1138+
1139+struct iopt_pages *iopt_alloc_pages(void __user *uptr, unsigned long length,
1140+ bool writable)
1141+{
1142+ struct iopt_pages *pages;
1143+
1144+ /*
1145+ * The iommu API uses size_t as the length, and protect the DIV_ROUND_UP
1146+ * below from overflow
1147+ */
1148+ if (length > SIZE_MAX - PAGE_SIZE || length == 0)
1149+ return ERR_PTR(-EINVAL);
1150+
1151+ pages = kzalloc(sizeof(*pages), GFP_KERNEL_ACCOUNT);
1152+ if (!pages)
1153+ return ERR_PTR(-ENOMEM);
1154+
1155+ kref_init(&pages->kref);
1156+ xa_init_flags(&pages->pinned_pfns, XA_FLAGS_ACCOUNT);
1157+ mutex_init(&pages->mutex);
1158+ pages->source_mm = current->mm;
1159+ mmgrab(pages->source_mm);
1160+ pages->uptr = (void __user *)ALIGN_DOWN((uintptr_t)uptr, PAGE_SIZE);
1161+ pages->npages = DIV_ROUND_UP(length + (uptr - pages->uptr), PAGE_SIZE);
1162+ pages->access_itree = RB_ROOT_CACHED;
1163+ pages->domains_itree = RB_ROOT_CACHED;
1164+ pages->writable = writable;
1165+ if (capable(CAP_IPC_LOCK))
1166+ pages->account_mode = IOPT_PAGES_ACCOUNT_NONE;
1167+ else
1168+ pages->account_mode = IOPT_PAGES_ACCOUNT_USER;
1169+ pages->source_task = current->group_leader;
1170+ get_task_struct(current->group_leader);
1171+ pages->source_user = get_uid(current_user());
1172+ return pages;
1173+}
1174+
1175+void iopt_release_pages(struct kref *kref)
1176+{
1177+ struct iopt_pages *pages = container_of(kref, struct iopt_pages, kref);
1178+
1179+ WARN_ON(!RB_EMPTY_ROOT(&pages->access_itree.rb_root));
1180+ WARN_ON(!RB_EMPTY_ROOT(&pages->domains_itree.rb_root));
1181+ WARN_ON(pages->npinned);
1182+ WARN_ON(!xa_empty(&pages->pinned_pfns));
1183+ mmdrop(pages->source_mm);
1184+ mutex_destroy(&pages->mutex);
1185+ put_task_struct(pages->source_task);
1186+ free_uid(pages->source_user);
1187+ kfree(pages);
1188+}
1189+
1190+static void
1191+iopt_area_unpin_domain(struct pfn_batch *batch, struct iopt_area *area,
1192+ struct iopt_pages *pages, struct iommu_domain *domain,
1193+ unsigned long start_index, unsigned long last_index,
1194+ unsigned long *unmapped_end_index,
1195+ unsigned long real_last_index)
1196+{
1197+ while (start_index <= last_index) {
1198+ unsigned long batch_last_index;
1199+
1200+ if (*unmapped_end_index <= last_index) {
1201+ unsigned long start =
1202+ max(start_index, *unmapped_end_index);
1203+
1204+ batch_from_domain(batch, domain, area, start,
1205+ last_index);
1206+ batch_last_index = start + batch->total_pfns - 1;
1207+ } else {
1208+ batch_last_index = last_index;
1209+ }
1210+
1211+ /*
1212+ * unmaps must always 'cut' at a place where the pfns are not
1213+ * contiguous to pair with the maps that always install
1214+ * contiguous pages. Thus, if we have to stop unpinning in the
1215+ * middle of the domains we need to keep reading pfns until we
1216+ * find a cut point to do the unmap. The pfns we read are
1217+ * carried over and either skipped or integrated into the next
1218+ * batch.
1219+ */
1220+ if (batch_last_index == last_index &&
1221+ last_index != real_last_index)
1222+ batch_from_domain_continue(batch, domain, area,
1223+ last_index + 1,
1224+ real_last_index);
1225+
1226+ if (*unmapped_end_index <= batch_last_index) {
1227+ iopt_area_unmap_domain_range(
1228+ area, domain, *unmapped_end_index,
1229+ start_index + batch->total_pfns - 1);
1230+ *unmapped_end_index = start_index + batch->total_pfns;
1231+ }
1232+
1233+ /* unpin must follow unmap */
1234+ batch_unpin(batch, pages, 0,
1235+ batch_last_index - start_index + 1);
1236+ start_index = batch_last_index + 1;
1237+
1238+ batch_clear_carry(batch,
1239+ *unmapped_end_index - batch_last_index - 1);
1240+ }
1241+}
1242+
1243+static void __iopt_area_unfill_domain(struct iopt_area *area,
1244+ struct iopt_pages *pages,
1245+ struct iommu_domain *domain,
1246+ unsigned long last_index)
1247+{
1248+ struct interval_tree_double_span_iter span;
1249+ unsigned long start_index = iopt_area_index(area);
1250+ unsigned long unmapped_end_index = start_index;
1251+ u64 backup[BATCH_BACKUP_SIZE];
1252+ struct pfn_batch batch;
1253+
1254+ lockdep_assert_held(&pages->mutex);
1255+
1256+ /*
1257+ * For security we must not unpin something that is still DMA mapped,
1258+ * so this must unmap any IOVA before we go ahead and unpin the pages.
1259+ * This creates a complexity where we need to skip over unpinning pages
1260+ * held in the xarray, but continue to unmap from the domain.
1261+ *
1262+ * The domain unmap cannot stop in the middle of a contiguous range of
1263+ * PFNs. To solve this problem the unpinning step will read ahead to the
1264+ * end of any contiguous span, unmap that whole span, and then only
1265+ * unpin the leading part that does not have any accesses. The residual
1266+ * PFNs that were unmapped but not unpinned are called a "carry" in the
1267+ * batch as they are moved to the front of the PFN list and continue on
1268+ * to the next iteration(s).
1269+ */
1270+ batch_init_backup(&batch, last_index + 1, backup, sizeof(backup));
1271+ interval_tree_for_each_double_span(&span, &pages->domains_itree,
1272+ &pages->access_itree, start_index,
1273+ last_index) {
1274+ if (span.is_used) {
1275+ batch_skip_carry(&batch,
1276+ span.last_used - span.start_used + 1);
1277+ continue;
1278+ }
1279+ iopt_area_unpin_domain(&batch, area, pages, domain,
1280+ span.start_hole, span.last_hole,
1281+ &unmapped_end_index, last_index);
1282+ }
1283+ /*
1284+ * If the range ends in a access then we do the residual unmap without
1285+ * any unpins.
1286+ */
1287+ if (unmapped_end_index != last_index + 1)
1288+ iopt_area_unmap_domain_range(area, domain, unmapped_end_index,
1289+ last_index);
1290+ WARN_ON(batch.total_pfns);
1291+ batch_destroy(&batch, backup);
1292+ update_unpinned(pages);
1293+}
1294+
1295+static void iopt_area_unfill_partial_domain(struct iopt_area *area,
1296+ struct iopt_pages *pages,
1297+ struct iommu_domain *domain,
1298+ unsigned long end_index)
1299+{
1300+ if (end_index != iopt_area_index(area))
1301+ __iopt_area_unfill_domain(area, pages, domain, end_index - 1);
1302+}
1303+
1304+/**
1305+ * iopt_area_unmap_domain() - Unmap without unpinning PFNs in a domain
1306+ * @area: The IOVA range to unmap
1307+ * @domain: The domain to unmap
1308+ *
1309+ * The caller must know that unpinning is not required, usually because there
1310+ * are other domains in the iopt.
1311+ */
1312+void iopt_area_unmap_domain(struct iopt_area *area, struct iommu_domain *domain)
1313+{
1314+ iommu_unmap_nofail(domain, iopt_area_iova(area),
1315+ iopt_area_length(area));
1316+}
1317+
1318+/**
1319+ * iopt_area_unfill_domain() - Unmap and unpin PFNs in a domain
1320+ * @area: IOVA area to use
1321+ * @pages: page supplier for the area (area->pages is NULL)
1322+ * @domain: Domain to unmap from
1323+ *
1324+ * The domain should be removed from the domains_itree before calling. The
1325+ * domain will always be unmapped, but the PFNs may not be unpinned if there are
1326+ * still accesses.
1327+ */
1328+void iopt_area_unfill_domain(struct iopt_area *area, struct iopt_pages *pages,
1329+ struct iommu_domain *domain)
1330+{
1331+ __iopt_area_unfill_domain(area, pages, domain,
1332+ iopt_area_last_index(area));
1333+}
1334+
1335+/**
1336+ * iopt_area_fill_domain() - Map PFNs from the area into a domain
1337+ * @area: IOVA area to use
1338+ * @domain: Domain to load PFNs into
1339+ *
1340+ * Read the pfns from the area's underlying iopt_pages and map them into the
1341+ * given domain. Called when attaching a new domain to an io_pagetable.
1342+ */
1343+int iopt_area_fill_domain(struct iopt_area *area, struct iommu_domain *domain)
1344+{
1345+ unsigned long done_end_index;
1346+ struct pfn_reader pfns;
1347+ int rc;
1348+
1349+ lockdep_assert_held(&area->pages->mutex);
1350+
1351+ rc = pfn_reader_first(&pfns, area->pages, iopt_area_index(area),
1352+ iopt_area_last_index(area));
1353+ if (rc)
1354+ return rc;
1355+
1356+ while (!pfn_reader_done(&pfns)) {
1357+ done_end_index = pfns.batch_start_index;
1358+ rc = batch_to_domain(&pfns.batch, domain, area,
1359+ pfns.batch_start_index);
1360+ if (rc)
1361+ goto out_unmap;
1362+ done_end_index = pfns.batch_end_index;
1363+
1364+ rc = pfn_reader_next(&pfns);
1365+ if (rc)
1366+ goto out_unmap;
1367+ }
1368+
1369+ rc = pfn_reader_update_pinned(&pfns);
1370+ if (rc)
1371+ goto out_unmap;
1372+ goto out_destroy;
1373+
1374+out_unmap:
1375+ pfn_reader_release_pins(&pfns);
1376+ iopt_area_unfill_partial_domain(area, area->pages, domain,
1377+ done_end_index);
1378+out_destroy:
1379+ pfn_reader_destroy(&pfns);
1380+ return rc;
1381+}
1382+
1383+/**
1384+ * iopt_area_fill_domains() - Install PFNs into the area's domains
1385+ * @area: The area to act on
1386+ * @pages: The pages associated with the area (area->pages is NULL)
1387+ *
1388+ * Called during area creation. The area is freshly created and not inserted in
1389+ * the domains_itree yet. PFNs are read and loaded into every domain held in the
1390+ * area's io_pagetable and the area is installed in the domains_itree.
1391+ *
1392+ * On failure all domains are left unchanged.
1393+ */
1394+int iopt_area_fill_domains(struct iopt_area *area, struct iopt_pages *pages)
1395+{
1396+ unsigned long done_first_end_index;
1397+ unsigned long done_all_end_index;
1398+ struct iommu_domain *domain;
1399+ unsigned long unmap_index;
1400+ struct pfn_reader pfns;
1401+ unsigned long index;
1402+ int rc;
1403+
1404+ lockdep_assert_held(&area->iopt->domains_rwsem);
1405+
1406+ if (xa_empty(&area->iopt->domains))
1407+ return 0;
1408+
1409+ mutex_lock(&pages->mutex);
1410+ rc = pfn_reader_first(&pfns, pages, iopt_area_index(area),
1411+ iopt_area_last_index(area));
1412+ if (rc)
1413+ goto out_unlock;
1414+
1415+ while (!pfn_reader_done(&pfns)) {
1416+ done_first_end_index = pfns.batch_end_index;
1417+ done_all_end_index = pfns.batch_start_index;
1418+ xa_for_each(&area->iopt->domains, index, domain) {
1419+ rc = batch_to_domain(&pfns.batch, domain, area,
1420+ pfns.batch_start_index);
1421+ if (rc)
1422+ goto out_unmap;
1423+ }
1424+ done_all_end_index = done_first_end_index;
1425+
1426+ rc = pfn_reader_next(&pfns);
1427+ if (rc)
1428+ goto out_unmap;
1429+ }
1430+ rc = pfn_reader_update_pinned(&pfns);
1431+ if (rc)
1432+ goto out_unmap;
1433+
1434+ area->storage_domain = xa_load(&area->iopt->domains, 0);
1435+ interval_tree_insert(&area->pages_node, &pages->domains_itree);
1436+ goto out_destroy;
1437+
1438+out_unmap:
1439+ pfn_reader_release_pins(&pfns);
1440+ xa_for_each(&area->iopt->domains, unmap_index, domain) {
1441+ unsigned long end_index;
1442+
1443+ if (unmap_index < index)
1444+ end_index = done_first_end_index;
1445+ else
1446+ end_index = done_all_end_index;
1447+
1448+ /*
1449+ * The area is not yet part of the domains_itree so we have to
1450+ * manage the unpinning specially. The last domain does the
1451+ * unpin, every other domain is just unmapped.
1452+ */
1453+ if (unmap_index != area->iopt->next_domain_id - 1) {
1454+ if (end_index != iopt_area_index(area))
1455+ iopt_area_unmap_domain_range(
1456+ area, domain, iopt_area_index(area),
1457+ end_index - 1);
1458+ } else {
1459+ iopt_area_unfill_partial_domain(area, pages, domain,
1460+ end_index);
1461+ }
1462+ }
1463+out_destroy:
1464+ pfn_reader_destroy(&pfns);
1465+out_unlock:
1466+ mutex_unlock(&pages->mutex);
1467+ return rc;
1468+}
1469+
1470+/**
1471+ * iopt_area_unfill_domains() - unmap PFNs from the area's domains
1472+ * @area: The area to act on
1473+ * @pages: The pages associated with the area (area->pages is NULL)
1474+ *
1475+ * Called during area destruction. This unmaps the iova's covered by all the
1476+ * area's domains and releases the PFNs.
1477+ */
1478+void iopt_area_unfill_domains(struct iopt_area *area, struct iopt_pages *pages)
1479+{
1480+ struct io_pagetable *iopt = area->iopt;
1481+ struct iommu_domain *domain;
1482+ unsigned long index;
1483+
1484+ lockdep_assert_held(&iopt->domains_rwsem);
1485+
1486+ mutex_lock(&pages->mutex);
1487+ if (!area->storage_domain)
1488+ goto out_unlock;
1489+
1490+ xa_for_each(&iopt->domains, index, domain)
1491+ if (domain != area->storage_domain)
1492+ iopt_area_unmap_domain_range(
1493+ area, domain, iopt_area_index(area),
1494+ iopt_area_last_index(area));
1495+
1496+ interval_tree_remove(&area->pages_node, &pages->domains_itree);
1497+ iopt_area_unfill_domain(area, pages, area->storage_domain);
1498+ area->storage_domain = NULL;
1499+out_unlock:
1500+ mutex_unlock(&pages->mutex);
1501+}
1502+
1503+static void iopt_pages_unpin_xarray(struct pfn_batch *batch,
1504+ struct iopt_pages *pages,
1505+ unsigned long start_index,
1506+ unsigned long end_index)
1507+{
1508+ while (start_index <= end_index) {
1509+ batch_from_xarray_clear(batch, &pages->pinned_pfns, start_index,
1510+ end_index);
1511+ batch_unpin(batch, pages, 0, batch->total_pfns);
1512+ start_index += batch->total_pfns;
1513+ batch_clear(batch);
1514+ }
1515+}
1516+
1517+/**
1518+ * iopt_pages_unfill_xarray() - Update the xarry after removing an access
1519+ * @pages: The pages to act on
1520+ * @start_index: Starting PFN index
1521+ * @last_index: Last PFN index
1522+ *
1523+ * Called when an iopt_pages_access is removed, removes pages from the itree.
1524+ * The access should already be removed from the access_itree.
1525+ */
1526+void iopt_pages_unfill_xarray(struct iopt_pages *pages,
1527+ unsigned long start_index,
1528+ unsigned long last_index)
1529+{
1530+ struct interval_tree_double_span_iter span;
1531+ u64 backup[BATCH_BACKUP_SIZE];
1532+ struct pfn_batch batch;
1533+ bool batch_inited = false;
1534+
1535+ lockdep_assert_held(&pages->mutex);
1536+
1537+ interval_tree_for_each_double_span(&span, &pages->access_itree,
1538+ &pages->domains_itree, start_index,
1539+ last_index) {
1540+ if (!span.is_used) {
1541+ if (!batch_inited) {
1542+ batch_init_backup(&batch,
1543+ last_index - start_index + 1,
1544+ backup, sizeof(backup));
1545+ batch_inited = true;
1546+ }
1547+ iopt_pages_unpin_xarray(&batch, pages, span.start_hole,
1548+ span.last_hole);
1549+ } else if (span.is_used == 2) {
1550+ /* Covered by a domain */
1551+ clear_xarray(&pages->pinned_pfns, span.start_used,
1552+ span.last_used);
1553+ }
1554+ /* Otherwise covered by an existing access */
1555+ }
1556+ if (batch_inited)
1557+ batch_destroy(&batch, backup);
1558+ update_unpinned(pages);
1559+}
1560+
1561+/**
1562+ * iopt_pages_fill_from_xarray() - Fast path for reading PFNs
1563+ * @pages: The pages to act on
1564+ * @start_index: The first page index in the range
1565+ * @last_index: The last page index in the range
1566+ * @out_pages: The output array to return the pages
1567+ *
1568+ * This can be called if the caller is holding a refcount on an
1569+ * iopt_pages_access that is known to have already been filled. It quickly reads
1570+ * the pages directly from the xarray.
1571+ *
1572+ * This is part of the SW iommu interface to read pages for in-kernel use.
1573+ */
1574+void iopt_pages_fill_from_xarray(struct iopt_pages *pages,
1575+ unsigned long start_index,
1576+ unsigned long last_index,
1577+ struct page **out_pages)
1578+{
1579+ XA_STATE(xas, &pages->pinned_pfns, start_index);
1580+ void *entry;
1581+
1582+ rcu_read_lock();
1583+ while (start_index <= last_index) {
1584+ entry = xas_next(&xas);
1585+ if (xas_retry(&xas, entry))
1586+ continue;
1587+ WARN_ON(!xa_is_value(entry));
1588+ *(out_pages++) = pfn_to_page(xa_to_value(entry));
1589+ start_index++;
1590+ }
1591+ rcu_read_unlock();
1592+}
1593+
1594+static int iopt_pages_fill_from_domain(struct iopt_pages *pages,
1595+ unsigned long start_index,
1596+ unsigned long last_index,
1597+ struct page **out_pages)
1598+{
1599+ while (start_index != last_index + 1) {
1600+ unsigned long domain_last;
1601+ struct iopt_area *area;
1602+
1603+ area = iopt_pages_find_domain_area(pages, start_index);
1604+ if (WARN_ON(!area))
1605+ return -EINVAL;
1606+
1607+ domain_last = min(iopt_area_last_index(area), last_index);
1608+ out_pages = raw_pages_from_domain(area->storage_domain, area,
1609+ start_index, domain_last,
1610+ out_pages);
1611+ start_index = domain_last + 1;
1612+ }
1613+ return 0;
1614+}
1615+
1616+static int iopt_pages_fill_from_mm(struct iopt_pages *pages,
1617+ struct pfn_reader_user *user,
1618+ unsigned long start_index,
1619+ unsigned long last_index,
1620+ struct page **out_pages)
1621+{
1622+ unsigned long cur_index = start_index;
1623+ int rc;
1624+
1625+ while (cur_index != last_index + 1) {
1626+ user->upages = out_pages + (cur_index - start_index);
1627+ rc = pfn_reader_user_pin(user, pages, cur_index, last_index);
1628+ if (rc)
1629+ goto out_unpin;
1630+ cur_index = user->upages_end;
1631+ }
1632+ return 0;
1633+
1634+out_unpin:
1635+ if (start_index != cur_index)
1636+ iopt_pages_err_unpin(pages, start_index, cur_index - 1,
1637+ out_pages);
1638+ return rc;
1639+}
1640+
1641+/**
1642+ * iopt_pages_fill_xarray() - Read PFNs
1643+ * @pages: The pages to act on
1644+ * @start_index: The first page index in the range
1645+ * @last_index: The last page index in the range
1646+ * @out_pages: The output array to return the pages, may be NULL
1647+ *
1648+ * This populates the xarray and returns the pages in out_pages. As the slow
1649+ * path this is able to copy pages from other storage tiers into the xarray.
1650+ *
1651+ * On failure the xarray is left unchanged.
1652+ *
1653+ * This is part of the SW iommu interface to read pages for in-kernel use.
1654+ */
1655+int iopt_pages_fill_xarray(struct iopt_pages *pages, unsigned long start_index,
1656+ unsigned long last_index, struct page **out_pages)
1657+{
1658+ struct interval_tree_double_span_iter span;
1659+ unsigned long xa_end = start_index;
1660+ struct pfn_reader_user user;
1661+ int rc;
1662+
1663+ lockdep_assert_held(&pages->mutex);
1664+
1665+ pfn_reader_user_init(&user, pages);
1666+ user.upages_len = (last_index - start_index + 1) * sizeof(*out_pages);
1667+ interval_tree_for_each_double_span(&span, &pages->access_itree,
1668+ &pages->domains_itree, start_index,
1669+ last_index) {
1670+ struct page **cur_pages;
1671+
1672+ if (span.is_used == 1) {
1673+ cur_pages = out_pages + (span.start_used - start_index);
1674+ iopt_pages_fill_from_xarray(pages, span.start_used,
1675+ span.last_used, cur_pages);
1676+ continue;
1677+ }
1678+
1679+ if (span.is_used == 2) {
1680+ cur_pages = out_pages + (span.start_used - start_index);
1681+ iopt_pages_fill_from_domain(pages, span.start_used,
1682+ span.last_used, cur_pages);
1683+ rc = pages_to_xarray(&pages->pinned_pfns,
1684+ span.start_used, span.last_used,
1685+ cur_pages);
1686+ if (rc)
1687+ goto out_clean_xa;
1688+ xa_end = span.last_used + 1;
1689+ continue;
1690+ }
1691+
1692+ /* hole */
1693+ cur_pages = out_pages + (span.start_hole - start_index);
1694+ rc = iopt_pages_fill_from_mm(pages, &user, span.start_hole,
1695+ span.last_hole, cur_pages);
1696+ if (rc)
1697+ goto out_clean_xa;
1698+ rc = pages_to_xarray(&pages->pinned_pfns, span.start_hole,
1699+ span.last_hole, cur_pages);
1700+ if (rc) {
1701+ iopt_pages_err_unpin(pages, span.start_hole,
1702+ span.last_hole, cur_pages);
1703+ goto out_clean_xa;
1704+ }
1705+ xa_end = span.last_hole + 1;
1706+ }
1707+ rc = pfn_reader_user_update_pinned(&user, pages);
1708+ if (rc)
1709+ goto out_clean_xa;
1710+ user.upages = NULL;
1711+ pfn_reader_user_destroy(&user, pages);
1712+ return 0;
1713+
1714+out_clean_xa:
1715+ if (start_index != xa_end)
1716+ iopt_pages_unfill_xarray(pages, start_index, xa_end - 1);
1717+ user.upages = NULL;
1718+ pfn_reader_user_destroy(&user, pages);
1719+ return rc;
1720+}
1721+
1722+/*
1723+ * This uses the pfn_reader instead of taking a shortcut by using the mm. It can
1724+ * do every scenario and is fully consistent with what an iommu_domain would
1725+ * see.
1726+ */
1727+static int iopt_pages_rw_slow(struct iopt_pages *pages,
1728+ unsigned long start_index,
1729+ unsigned long last_index, unsigned long offset,
1730+ void *data, unsigned long length,
1731+ unsigned int flags)
1732+{
1733+ struct pfn_reader pfns;
1734+ int rc;
1735+
1736+ mutex_lock(&pages->mutex);
1737+
1738+ rc = pfn_reader_first(&pfns, pages, start_index, last_index);
1739+ if (rc)
1740+ goto out_unlock;
1741+
1742+ while (!pfn_reader_done(&pfns)) {
1743+ unsigned long done;
1744+
1745+ done = batch_rw(&pfns.batch, data, offset, length, flags);
1746+ data += done;
1747+ length -= done;
1748+ offset = 0;
1749+ pfn_reader_unpin(&pfns);
1750+
1751+ rc = pfn_reader_next(&pfns);
1752+ if (rc)
1753+ goto out_destroy;
1754+ }
1755+ if (WARN_ON(length != 0))
1756+ rc = -EINVAL;
1757+out_destroy:
1758+ pfn_reader_destroy(&pfns);
1759+out_unlock:
1760+ mutex_unlock(&pages->mutex);
1761+ return rc;
1762+}
1763+
1764+/*
1765+ * A medium speed path that still allows DMA inconsistencies, but doesn't do any
1766+ * memory allocations or interval tree searches.
1767+ */
1768+static int iopt_pages_rw_page(struct iopt_pages *pages, unsigned long index,
1769+ unsigned long offset, void *data,
1770+ unsigned long length, unsigned int flags)
1771+{
1772+ struct page *page = NULL;
1773+ int rc;
1774+
1775+ if (!mmget_not_zero(pages->source_mm))
1776+ return iopt_pages_rw_slow(pages, index, index, offset, data,
1777+ length, flags);
1778+
1779+ if (iommufd_should_fail()) {
1780+ rc = -EINVAL;
1781+ goto out_mmput;
1782+ }
1783+
1784+ mmap_read_lock(pages->source_mm);
1785+ rc = pin_user_pages_remote(
1786+ pages->source_mm, (uintptr_t)(pages->uptr + index * PAGE_SIZE),
1787+ 1, (flags & IOMMUFD_ACCESS_RW_WRITE) ? FOLL_WRITE : 0, &page,
1788+ NULL, NULL);
1789+ mmap_read_unlock(pages->source_mm);
1790+ if (rc != 1) {
1791+ if (WARN_ON(rc >= 0))
1792+ rc = -EINVAL;
1793+ goto out_mmput;
1794+ }
1795+ copy_data_page(page, data, offset, length, flags);
1796+ unpin_user_page(page);
1797+ rc = 0;
1798+
1799+out_mmput:
1800+ mmput(pages->source_mm);
1801+ return rc;
1802+}
1803+
1804+/**
1805+ * iopt_pages_rw_access - Copy to/from a linear slice of the pages
1806+ * @pages: pages to act on
1807+ * @start_byte: First byte of pages to copy to/from
1808+ * @data: Kernel buffer to get/put the data
1809+ * @length: Number of bytes to copy
1810+ * @flags: IOMMUFD_ACCESS_RW_* flags
1811+ *
1812+ * This will find each page in the range, kmap it and then memcpy to/from
1813+ * the given kernel buffer.
1814+ */
1815+int iopt_pages_rw_access(struct iopt_pages *pages, unsigned long start_byte,
1816+ void *data, unsigned long length, unsigned int flags)
1817+{
1818+ unsigned long start_index = start_byte / PAGE_SIZE;
1819+ unsigned long last_index = (start_byte + length - 1) / PAGE_SIZE;
1820+ bool change_mm = current->mm != pages->source_mm;
1821+ int rc = 0;
1822+
1823+ if (IS_ENABLED(CONFIG_IOMMUFD_TEST) &&
1824+ (flags & __IOMMUFD_ACCESS_RW_SLOW_PATH))
1825+ change_mm = true;
1826+
1827+ if ((flags & IOMMUFD_ACCESS_RW_WRITE) && !pages->writable)
1828+ return -EPERM;
1829+
1830+ if (!(flags & IOMMUFD_ACCESS_RW_KTHREAD) && change_mm) {
1831+ if (start_index == last_index)
1832+ return iopt_pages_rw_page(pages, start_index,
1833+ start_byte % PAGE_SIZE, data,
1834+ length, flags);
1835+ return iopt_pages_rw_slow(pages, start_index, last_index,
1836+ start_byte % PAGE_SIZE, data, length,
1837+ flags);
1838+ }
1839+
1840+ /*
1841+ * Try to copy using copy_to_user(). We do this as a fast path and
1842+ * ignore any pinning inconsistencies, unlike a real DMA path.
1843+ */
1844+ if (change_mm) {
1845+ if (!mmget_not_zero(pages->source_mm))
1846+ return iopt_pages_rw_slow(pages, start_index,
1847+ last_index,
1848+ start_byte % PAGE_SIZE, data,
1849+ length, flags);
1850+ kthread_use_mm(pages->source_mm);
1851+ }
1852+
1853+ if (flags & IOMMUFD_ACCESS_RW_WRITE) {
1854+ if (copy_to_user(pages->uptr + start_byte, data, length))
1855+ rc = -EFAULT;
1856+ } else {
1857+ if (copy_from_user(data, pages->uptr + start_byte, length))
1858+ rc = -EFAULT;
1859+ }
1860+
1861+ if (change_mm) {
1862+ kthread_unuse_mm(pages->source_mm);
1863+ mmput(pages->source_mm);
1864+ }
1865+
1866+ return rc;
1867+}
1868+
1869+static struct iopt_pages_access *
1870+iopt_pages_get_exact_access(struct iopt_pages *pages, unsigned long index,
1871+ unsigned long last)
1872+{
1873+ struct interval_tree_node *node;
1874+
1875+ lockdep_assert_held(&pages->mutex);
1876+
1877+ /* There can be overlapping ranges in this interval tree */
1878+ for (node = interval_tree_iter_first(&pages->access_itree, index, last);
1879+ node; node = interval_tree_iter_next(node, index, last))
1880+ if (node->start == index && node->last == last)
1881+ return container_of(node, struct iopt_pages_access,
1882+ node);
1883+ return NULL;
1884+}
1885+
1886+/**
1887+ * iopt_area_add_access() - Record an in-knerel access for PFNs
1888+ * @area: The source of PFNs
1889+ * @start_index: First page index
1890+ * @last_index: Inclusive last page index
1891+ * @out_pages: Output list of struct page's representing the PFNs
1892+ * @flags: IOMMUFD_ACCESS_RW_* flags
1893+ *
1894+ * Record that an in-kernel access will be accessing the pages, ensure they are
1895+ * pinned, and return the PFNs as a simple list of 'struct page *'.
1896+ *
1897+ * This should be undone through a matching call to iopt_area_remove_access()
1898+ */
1899+int iopt_area_add_access(struct iopt_area *area, unsigned long start_index,
1900+ unsigned long last_index, struct page **out_pages,
1901+ unsigned int flags)
1902+{
1903+ struct iopt_pages *pages = area->pages;
1904+ struct iopt_pages_access *access;
1905+ int rc;
1906+
1907+ if ((flags & IOMMUFD_ACCESS_RW_WRITE) && !pages->writable)
1908+ return -EPERM;
1909+
1910+ mutex_lock(&pages->mutex);
1911+ access = iopt_pages_get_exact_access(pages, start_index, last_index);
1912+ if (access) {
1913+ area->num_accesses++;
1914+ access->users++;
1915+ iopt_pages_fill_from_xarray(pages, start_index, last_index,
1916+ out_pages);
1917+ mutex_unlock(&pages->mutex);
1918+ return 0;
1919+ }
1920+
1921+ access = kzalloc(sizeof(*access), GFP_KERNEL_ACCOUNT);
1922+ if (!access) {
1923+ rc = -ENOMEM;
1924+ goto err_unlock;
1925+ }
1926+
1927+ rc = iopt_pages_fill_xarray(pages, start_index, last_index, out_pages);
1928+ if (rc)
1929+ goto err_free;
1930+
1931+ access->node.start = start_index;
1932+ access->node.last = last_index;
1933+ access->users = 1;
1934+ area->num_accesses++;
1935+ interval_tree_insert(&access->node, &pages->access_itree);
1936+ mutex_unlock(&pages->mutex);
1937+ return 0;
1938+
1939+err_free:
1940+ kfree(access);
1941+err_unlock:
1942+ mutex_unlock(&pages->mutex);
1943+ return rc;
1944+}
1945+
1946+/**
1947+ * iopt_area_remove_access() - Release an in-kernel access for PFNs
1948+ * @area: The source of PFNs
1949+ * @start_index: First page index
1950+ * @last_index: Inclusive last page index
1951+ *
1952+ * Undo iopt_area_add_access() and unpin the pages if necessary. The caller
1953+ * must stop using the PFNs before calling this.
1954+ */
1955+void iopt_area_remove_access(struct iopt_area *area, unsigned long start_index,
1956+ unsigned long last_index)
1957+{
1958+ struct iopt_pages *pages = area->pages;
1959+ struct iopt_pages_access *access;
1960+
1961+ mutex_lock(&pages->mutex);
1962+ access = iopt_pages_get_exact_access(pages, start_index, last_index);
1963+ if (WARN_ON(!access))
1964+ goto out_unlock;
1965+
1966+ WARN_ON(area->num_accesses == 0 || access->users == 0);
1967+ area->num_accesses--;
1968+ access->users--;
1969+ if (access->users)
1970+ goto out_unlock;
1971+
1972+ interval_tree_remove(&access->node, &pages->access_itree);
1973+ iopt_pages_unfill_xarray(pages, start_index, last_index);
1974+ kfree(access);
1975+out_unlock:
1976+ mutex_unlock(&pages->mutex);
1977+}
drivers/iommu/iommufd/selftest.c+853−0
@@ -0,0 +1,853 @@
1+// SPDX-License-Identifier: GPL-2.0
2+/* Copyright (c) 2021-2022, NVIDIA CORPORATION & AFFILIATES.
3+ *
4+ * Kernel side components to support tools/testing/selftests/iommu
5+ */
6+#include <linux/slab.h>
7+#include <linux/iommu.h>
8+#include <linux/xarray.h>
9+#include <linux/file.h>
10+#include <linux/anon_inodes.h>
11+#include <linux/fault-inject.h>
12+#include <uapi/linux/iommufd.h>
13+
14+#include "io_pagetable.h"
15+#include "iommufd_private.h"
16+#include "iommufd_test.h"
17+
18+static DECLARE_FAULT_ATTR(fail_iommufd);
19+static struct dentry *dbgfs_root;
20+
21+size_t iommufd_test_memory_limit = 65536;
22+
23+enum {
24+ MOCK_IO_PAGE_SIZE = PAGE_SIZE / 2,
25+
26+ /*
27+ * Like a real page table alignment requires the low bits of the address
28+ * to be zero. xarray also requires the high bit to be zero, so we store
29+ * the pfns shifted. The upper bits are used for metadata.
30+ */
31+ MOCK_PFN_MASK = ULONG_MAX / MOCK_IO_PAGE_SIZE,
32+
33+ _MOCK_PFN_START = MOCK_PFN_MASK + 1,
34+ MOCK_PFN_START_IOVA = _MOCK_PFN_START,
35+ MOCK_PFN_LAST_IOVA = _MOCK_PFN_START,
36+};
37+
38+/*
39+ * Syzkaller has trouble randomizing the correct iova to use since it is linked
40+ * to the map ioctl's output, and it has no ide about that. So, simplify things.
41+ * In syzkaller mode the 64 bit IOVA is converted into an nth area and offset
42+ * value. This has a much smaller randomization space and syzkaller can hit it.
43+ */
44+static unsigned long iommufd_test_syz_conv_iova(struct io_pagetable *iopt,
45+ u64 *iova)
46+{
47+ struct syz_layout {
48+ __u32 nth_area;
49+ __u32 offset;
50+ };
51+ struct syz_layout *syz = (void *)iova;
52+ unsigned int nth = syz->nth_area;
53+ struct iopt_area *area;
54+
55+ down_read(&iopt->iova_rwsem);
56+ for (area = iopt_area_iter_first(iopt, 0, ULONG_MAX); area;
57+ area = iopt_area_iter_next(area, 0, ULONG_MAX)) {
58+ if (nth == 0) {
59+ up_read(&iopt->iova_rwsem);
60+ return iopt_area_iova(area) + syz->offset;
61+ }
62+ nth--;
63+ }
64+ up_read(&iopt->iova_rwsem);
65+
66+ return 0;
67+}
68+
69+void iommufd_test_syz_conv_iova_id(struct iommufd_ucmd *ucmd,
70+ unsigned int ioas_id, u64 *iova, u32 *flags)
71+{
72+ struct iommufd_ioas *ioas;
73+
74+ if (!(*flags & MOCK_FLAGS_ACCESS_SYZ))
75+ return;
76+ *flags &= ~(u32)MOCK_FLAGS_ACCESS_SYZ;
77+
78+ ioas = iommufd_get_ioas(ucmd, ioas_id);
79+ if (IS_ERR(ioas))
80+ return;
81+ *iova = iommufd_test_syz_conv_iova(&ioas->iopt, iova);
82+ iommufd_put_object(&ioas->obj);
83+}
84+
85+struct mock_iommu_domain {
86+ struct iommu_domain domain;
87+ struct xarray pfns;
88+};
89+
90+enum selftest_obj_type {
91+ TYPE_IDEV,
92+};
93+
94+struct selftest_obj {
95+ struct iommufd_object obj;
96+ enum selftest_obj_type type;
97+
98+ union {
99+ struct {
100+ struct iommufd_hw_pagetable *hwpt;
101+ struct iommufd_ctx *ictx;
102+ struct device mock_dev;
103+ } idev;
104+ };
105+};
106+
107+static struct iommu_domain *mock_domain_alloc(unsigned int iommu_domain_type)
108+{
109+ struct mock_iommu_domain *mock;
110+
111+ if (WARN_ON(iommu_domain_type != IOMMU_DOMAIN_UNMANAGED))
112+ return NULL;
113+
114+ mock = kzalloc(sizeof(*mock), GFP_KERNEL);
115+ if (!mock)
116+ return NULL;
117+ mock->domain.geometry.aperture_start = MOCK_APERTURE_START;
118+ mock->domain.geometry.aperture_end = MOCK_APERTURE_LAST;
119+ mock->domain.pgsize_bitmap = MOCK_IO_PAGE_SIZE;
120+ xa_init(&mock->pfns);
121+ return &mock->domain;
122+}
123+
124+static void mock_domain_free(struct iommu_domain *domain)
125+{
126+ struct mock_iommu_domain *mock =
127+ container_of(domain, struct mock_iommu_domain, domain);
128+
129+ WARN_ON(!xa_empty(&mock->pfns));
130+ kfree(mock);
131+}
132+
133+static int mock_domain_map_pages(struct iommu_domain *domain,
134+ unsigned long iova, phys_addr_t paddr,
135+ size_t pgsize, size_t pgcount, int prot,
136+ gfp_t gfp, size_t *mapped)
137+{
138+ struct mock_iommu_domain *mock =
139+ container_of(domain, struct mock_iommu_domain, domain);
140+ unsigned long flags = MOCK_PFN_START_IOVA;
141+ unsigned long start_iova = iova;
142+
143+ /*
144+ * xarray does not reliably work with fault injection because it does a
145+ * retry allocation, so put our own failure point.
146+ */
147+ if (iommufd_should_fail())
148+ return -ENOENT;
149+
150+ WARN_ON(iova % MOCK_IO_PAGE_SIZE);
151+ WARN_ON(pgsize % MOCK_IO_PAGE_SIZE);
152+ for (; pgcount; pgcount--) {
153+ size_t cur;
154+
155+ for (cur = 0; cur != pgsize; cur += MOCK_IO_PAGE_SIZE) {
156+ void *old;
157+
158+ if (pgcount == 1 && cur + MOCK_IO_PAGE_SIZE == pgsize)
159+ flags = MOCK_PFN_LAST_IOVA;
160+ old = xa_store(&mock->pfns, iova / MOCK_IO_PAGE_SIZE,
161+ xa_mk_value((paddr / MOCK_IO_PAGE_SIZE) |
162+ flags),
163+ gfp);
164+ if (xa_is_err(old)) {
165+ for (; start_iova != iova;
166+ start_iova += MOCK_IO_PAGE_SIZE)
167+ xa_erase(&mock->pfns,
168+ start_iova /
169+ MOCK_IO_PAGE_SIZE);
170+ return xa_err(old);
171+ }
172+ WARN_ON(old);
173+ iova += MOCK_IO_PAGE_SIZE;
174+ paddr += MOCK_IO_PAGE_SIZE;
175+ *mapped += MOCK_IO_PAGE_SIZE;
176+ flags = 0;
177+ }
178+ }
179+ return 0;
180+}
181+
182+static size_t mock_domain_unmap_pages(struct iommu_domain *domain,
183+ unsigned long iova, size_t pgsize,
184+ size_t pgcount,
185+ struct iommu_iotlb_gather *iotlb_gather)
186+{
187+ struct mock_iommu_domain *mock =
188+ container_of(domain, struct mock_iommu_domain, domain);
189+ bool first = true;
190+ size_t ret = 0;
191+ void *ent;
192+
193+ WARN_ON(iova % MOCK_IO_PAGE_SIZE);
194+ WARN_ON(pgsize % MOCK_IO_PAGE_SIZE);
195+
196+ for (; pgcount; pgcount--) {
197+ size_t cur;
198+
199+ for (cur = 0; cur != pgsize; cur += MOCK_IO_PAGE_SIZE) {
200+ ent = xa_erase(&mock->pfns, iova / MOCK_IO_PAGE_SIZE);
201+ WARN_ON(!ent);
202+ /*
203+ * iommufd generates unmaps that must be a strict
204+ * superset of the map's performend So every starting
205+ * IOVA should have been an iova passed to map, and the
206+ *
207+ * First IOVA must be present and have been a first IOVA
208+ * passed to map_pages
209+ */
210+ if (first) {
211+ WARN_ON(!(xa_to_value(ent) &
212+ MOCK_PFN_START_IOVA));
213+ first = false;
214+ }
215+ if (pgcount == 1 && cur + MOCK_IO_PAGE_SIZE == pgsize)
216+ WARN_ON(!(xa_to_value(ent) &
217+ MOCK_PFN_LAST_IOVA));
218+
219+ iova += MOCK_IO_PAGE_SIZE;
220+ ret += MOCK_IO_PAGE_SIZE;
221+ }
222+ }
223+ return ret;
224+}
225+
226+static phys_addr_t mock_domain_iova_to_phys(struct iommu_domain *domain,
227+ dma_addr_t iova)
228+{
229+ struct mock_iommu_domain *mock =
230+ container_of(domain, struct mock_iommu_domain, domain);
231+ void *ent;
232+
233+ WARN_ON(iova % MOCK_IO_PAGE_SIZE);
234+ ent = xa_load(&mock->pfns, iova / MOCK_IO_PAGE_SIZE);
235+ WARN_ON(!ent);
236+ return (xa_to_value(ent) & MOCK_PFN_MASK) * MOCK_IO_PAGE_SIZE;
237+}
238+
239+static const struct iommu_ops mock_ops = {
240+ .owner = THIS_MODULE,
241+ .pgsize_bitmap = MOCK_IO_PAGE_SIZE,
242+ .domain_alloc = mock_domain_alloc,
243+ .default_domain_ops =
244+ &(struct iommu_domain_ops){
245+ .free = mock_domain_free,
246+ .map_pages = mock_domain_map_pages,
247+ .unmap_pages = mock_domain_unmap_pages,
248+ .iova_to_phys = mock_domain_iova_to_phys,
249+ },
250+};
251+
252+static inline struct iommufd_hw_pagetable *
253+get_md_pagetable(struct iommufd_ucmd *ucmd, u32 mockpt_id,
254+ struct mock_iommu_domain **mock)
255+{
256+ struct iommufd_hw_pagetable *hwpt;
257+ struct iommufd_object *obj;
258+
259+ obj = iommufd_get_object(ucmd->ictx, mockpt_id,
260+ IOMMUFD_OBJ_HW_PAGETABLE);
261+ if (IS_ERR(obj))
262+ return ERR_CAST(obj);
263+ hwpt = container_of(obj, struct iommufd_hw_pagetable, obj);
264+ if (hwpt->domain->ops != mock_ops.default_domain_ops) {
265+ iommufd_put_object(&hwpt->obj);
266+ return ERR_PTR(-EINVAL);
267+ }
268+ *mock = container_of(hwpt->domain, struct mock_iommu_domain, domain);
269+ return hwpt;
270+}
271+
272+/* Create an hw_pagetable with the mock domain so we can test the domain ops */
273+static int iommufd_test_mock_domain(struct iommufd_ucmd *ucmd,
274+ struct iommu_test_cmd *cmd)
275+{
276+ static struct bus_type mock_bus = { .iommu_ops = &mock_ops };
277+ struct iommufd_hw_pagetable *hwpt;
278+ struct selftest_obj *sobj;
279+ struct iommufd_ioas *ioas;
280+ int rc;
281+
282+ ioas = iommufd_get_ioas(ucmd, cmd->id);
283+ if (IS_ERR(ioas))
284+ return PTR_ERR(ioas);
285+
286+ sobj = iommufd_object_alloc(ucmd->ictx, sobj, IOMMUFD_OBJ_SELFTEST);
287+ if (IS_ERR(sobj)) {
288+ rc = PTR_ERR(sobj);
289+ goto out_ioas;
290+ }
291+ sobj->idev.ictx = ucmd->ictx;
292+ sobj->type = TYPE_IDEV;
293+ sobj->idev.mock_dev.bus = &mock_bus;
294+
295+ hwpt = iommufd_device_selftest_attach(ucmd->ictx, ioas,
296+ &sobj->idev.mock_dev);
297+ if (IS_ERR(hwpt)) {
298+ rc = PTR_ERR(hwpt);
299+ goto out_sobj;
300+ }
301+ sobj->idev.hwpt = hwpt;
302+
303+ /* Userspace must destroy both of these IDs to destroy the object */
304+ cmd->mock_domain.out_hwpt_id = hwpt->obj.id;
305+ cmd->mock_domain.out_device_id = sobj->obj.id;
306+ iommufd_object_finalize(ucmd->ictx, &sobj->obj);
307+ iommufd_put_object(&ioas->obj);
308+ return iommufd_ucmd_respond(ucmd, sizeof(*cmd));
309+
310+out_sobj:
311+ iommufd_object_abort(ucmd->ictx, &sobj->obj);
312+out_ioas:
313+ iommufd_put_object(&ioas->obj);
314+ return rc;
315+}
316+
317+/* Add an additional reserved IOVA to the IOAS */
318+static int iommufd_test_add_reserved(struct iommufd_ucmd *ucmd,
319+ unsigned int mockpt_id,
320+ unsigned long start, size_t length)
321+{
322+ struct iommufd_ioas *ioas;
323+ int rc;
324+
325+ ioas = iommufd_get_ioas(ucmd, mockpt_id);
326+ if (IS_ERR(ioas))
327+ return PTR_ERR(ioas);
328+ down_write(&ioas->iopt.iova_rwsem);
329+ rc = iopt_reserve_iova(&ioas->iopt, start, start + length - 1, NULL);
330+ up_write(&ioas->iopt.iova_rwsem);
331+ iommufd_put_object(&ioas->obj);
332+ return rc;
333+}
334+
335+/* Check that every pfn under each iova matches the pfn under a user VA */
336+static int iommufd_test_md_check_pa(struct iommufd_ucmd *ucmd,
337+ unsigned int mockpt_id, unsigned long iova,
338+ size_t length, void __user *uptr)
339+{
340+ struct iommufd_hw_pagetable *hwpt;
341+ struct mock_iommu_domain *mock;
342+ int rc;
343+
344+ if (iova % MOCK_IO_PAGE_SIZE || length % MOCK_IO_PAGE_SIZE ||
345+ (uintptr_t)uptr % MOCK_IO_PAGE_SIZE)
346+ return -EINVAL;
347+
348+ hwpt = get_md_pagetable(ucmd, mockpt_id, &mock);
349+ if (IS_ERR(hwpt))
350+ return PTR_ERR(hwpt);
351+
352+ for (; length; length -= MOCK_IO_PAGE_SIZE) {
353+ struct page *pages[1];
354+ unsigned long pfn;
355+ long npages;
356+ void *ent;
357+
358+ npages = get_user_pages_fast((uintptr_t)uptr & PAGE_MASK, 1, 0,
359+ pages);
360+ if (npages < 0) {
361+ rc = npages;
362+ goto out_put;
363+ }
364+ if (WARN_ON(npages != 1)) {
365+ rc = -EFAULT;
366+ goto out_put;
367+ }
368+ pfn = page_to_pfn(pages[0]);
369+ put_page(pages[0]);
370+
371+ ent = xa_load(&mock->pfns, iova / MOCK_IO_PAGE_SIZE);
372+ if (!ent ||
373+ (xa_to_value(ent) & MOCK_PFN_MASK) * MOCK_IO_PAGE_SIZE !=
374+ pfn * PAGE_SIZE + ((uintptr_t)uptr % PAGE_SIZE)) {
375+ rc = -EINVAL;
376+ goto out_put;
377+ }
378+ iova += MOCK_IO_PAGE_SIZE;
379+ uptr += MOCK_IO_PAGE_SIZE;
380+ }
381+ rc = 0;
382+
383+out_put:
384+ iommufd_put_object(&hwpt->obj);
385+ return rc;
386+}
387+
388+/* Check that the page ref count matches, to look for missing pin/unpins */
389+static int iommufd_test_md_check_refs(struct iommufd_ucmd *ucmd,
390+ void __user *uptr, size_t length,
391+ unsigned int refs)
392+{
393+ if (length % PAGE_SIZE || (uintptr_t)uptr % PAGE_SIZE)
394+ return -EINVAL;
395+
396+ for (; length; length -= PAGE_SIZE) {
397+ struct page *pages[1];
398+ long npages;
399+
400+ npages = get_user_pages_fast((uintptr_t)uptr, 1, 0, pages);
401+ if (npages < 0)
402+ return npages;
403+ if (WARN_ON(npages != 1))
404+ return -EFAULT;
405+ if (!PageCompound(pages[0])) {
406+ unsigned int count;
407+
408+ count = page_ref_count(pages[0]);
409+ if (count / GUP_PIN_COUNTING_BIAS != refs) {
410+ put_page(pages[0]);
411+ return -EIO;
412+ }
413+ }
414+ put_page(pages[0]);
415+ uptr += PAGE_SIZE;
416+ }
417+ return 0;
418+}
419+
420+struct selftest_access {
421+ struct iommufd_access *access;
422+ struct file *file;
423+ struct mutex lock;
424+ struct list_head items;
425+ unsigned int next_id;
426+ bool destroying;
427+};
428+
429+struct selftest_access_item {
430+ struct list_head items_elm;
431+ unsigned long iova;
432+ size_t length;
433+ unsigned int id;
434+};
435+
436+static const struct file_operations iommfd_test_staccess_fops;
437+
438+static struct selftest_access *iommufd_access_get(int fd)
439+{
440+ struct file *file;
441+
442+ file = fget(fd);
443+ if (!file)
444+ return ERR_PTR(-EBADFD);
445+
446+ if (file->f_op != &iommfd_test_staccess_fops) {
447+ fput(file);
448+ return ERR_PTR(-EBADFD);
449+ }
450+ return file->private_data;
451+}
452+
453+static void iommufd_test_access_unmap(void *data, unsigned long iova,
454+ unsigned long length)
455+{
456+ unsigned long iova_last = iova + length - 1;
457+ struct selftest_access *staccess = data;
458+ struct selftest_access_item *item;
459+ struct selftest_access_item *tmp;
460+
461+ mutex_lock(&staccess->lock);
462+ list_for_each_entry_safe(item, tmp, &staccess->items, items_elm) {
463+ if (iova > item->iova + item->length - 1 ||
464+ iova_last < item->iova)
465+ continue;
466+ list_del(&item->items_elm);
467+ iommufd_access_unpin_pages(staccess->access, item->iova,
468+ item->length);
469+ kfree(item);
470+ }
471+ mutex_unlock(&staccess->lock);
472+}
473+
474+static int iommufd_test_access_item_destroy(struct iommufd_ucmd *ucmd,
475+ unsigned int access_id,
476+ unsigned int item_id)
477+{
478+ struct selftest_access_item *item;
479+ struct selftest_access *staccess;
480+
481+ staccess = iommufd_access_get(access_id);
482+ if (IS_ERR(staccess))
483+ return PTR_ERR(staccess);
484+
485+ mutex_lock(&staccess->lock);
486+ list_for_each_entry(item, &staccess->items, items_elm) {
487+ if (item->id == item_id) {
488+ list_del(&item->items_elm);
489+ iommufd_access_unpin_pages(staccess->access, item->iova,
490+ item->length);
491+ mutex_unlock(&staccess->lock);
492+ kfree(item);
493+ fput(staccess->file);
494+ return 0;
495+ }
496+ }
497+ mutex_unlock(&staccess->lock);
498+ fput(staccess->file);
499+ return -ENOENT;
500+}
501+
502+static int iommufd_test_staccess_release(struct inode *inode,
503+ struct file *filep)
504+{
505+ struct selftest_access *staccess = filep->private_data;
506+
507+ if (staccess->access) {
508+ iommufd_test_access_unmap(staccess, 0, ULONG_MAX);
509+ iommufd_access_destroy(staccess->access);
510+ }
511+ mutex_destroy(&staccess->lock);
512+ kfree(staccess);
513+ return 0;
514+}
515+
516+static const struct iommufd_access_ops selftest_access_ops_pin = {
517+ .needs_pin_pages = 1,
518+ .unmap = iommufd_test_access_unmap,
519+};
520+
521+static const struct iommufd_access_ops selftest_access_ops = {
522+ .unmap = iommufd_test_access_unmap,
523+};
524+
525+static const struct file_operations iommfd_test_staccess_fops = {
526+ .release = iommufd_test_staccess_release,
527+};
528+
529+static struct selftest_access *iommufd_test_alloc_access(void)
530+{
531+ struct selftest_access *staccess;
532+ struct file *filep;
533+
534+ staccess = kzalloc(sizeof(*staccess), GFP_KERNEL_ACCOUNT);
535+ if (!staccess)
536+ return ERR_PTR(-ENOMEM);
537+ INIT_LIST_HEAD(&staccess->items);
538+ mutex_init(&staccess->lock);
539+
540+ filep = anon_inode_getfile("[iommufd_test_staccess]",
541+ &iommfd_test_staccess_fops, staccess,
542+ O_RDWR);
543+ if (IS_ERR(filep)) {
544+ kfree(staccess);
545+ return ERR_CAST(filep);
546+ }
547+ staccess->file = filep;
548+ return staccess;
549+}
550+
551+static int iommufd_test_create_access(struct iommufd_ucmd *ucmd,
552+ unsigned int ioas_id, unsigned int flags)
553+{
554+ struct iommu_test_cmd *cmd = ucmd->cmd;
555+ struct selftest_access *staccess;
556+ struct iommufd_access *access;
557+ int fdno;
558+ int rc;
559+
560+ if (flags & ~MOCK_FLAGS_ACCESS_CREATE_NEEDS_PIN_PAGES)
561+ return -EOPNOTSUPP;
562+
563+ staccess = iommufd_test_alloc_access();
564+ if (IS_ERR(staccess))
565+ return PTR_ERR(staccess);
566+
567+ fdno = get_unused_fd_flags(O_CLOEXEC);
568+ if (fdno < 0) {
569+ rc = -ENOMEM;
570+ goto out_free_staccess;
571+ }
572+
573+ access = iommufd_access_create(
574+ ucmd->ictx, ioas_id,
575+ (flags & MOCK_FLAGS_ACCESS_CREATE_NEEDS_PIN_PAGES) ?
576+ &selftest_access_ops_pin :
577+ &selftest_access_ops,
578+ staccess);
579+ if (IS_ERR(access)) {
580+ rc = PTR_ERR(access);
581+ goto out_put_fdno;
582+ }
583+ cmd->create_access.out_access_fd = fdno;
584+ rc = iommufd_ucmd_respond(ucmd, sizeof(*cmd));
585+ if (rc)
586+ goto out_destroy;
587+
588+ staccess->access = access;
589+ fd_install(fdno, staccess->file);
590+ return 0;
591+
592+out_destroy:
593+ iommufd_access_destroy(access);
594+out_put_fdno:
595+ put_unused_fd(fdno);
596+out_free_staccess:
597+ fput(staccess->file);
598+ return rc;
599+}
600+
601+/* Check that the pages in a page array match the pages in the user VA */
602+static int iommufd_test_check_pages(void __user *uptr, struct page **pages,
603+ size_t npages)
604+{
605+ for (; npages; npages--) {
606+ struct page *tmp_pages[1];
607+ long rc;
608+
609+ rc = get_user_pages_fast((uintptr_t)uptr, 1, 0, tmp_pages);
610+ if (rc < 0)
611+ return rc;
612+ if (WARN_ON(rc != 1))
613+ return -EFAULT;
614+ put_page(tmp_pages[0]);
615+ if (tmp_pages[0] != *pages)
616+ return -EBADE;
617+ pages++;
618+ uptr += PAGE_SIZE;
619+ }
620+ return 0;
621+}
622+
623+static int iommufd_test_access_pages(struct iommufd_ucmd *ucmd,
624+ unsigned int access_id, unsigned long iova,
625+ size_t length, void __user *uptr,
626+ u32 flags)
627+{
628+ struct iommu_test_cmd *cmd = ucmd->cmd;
629+ struct selftest_access_item *item;
630+ struct selftest_access *staccess;
631+ struct page **pages;
632+ size_t npages;
633+ int rc;
634+
635+ /* Prevent syzkaller from triggering a WARN_ON in kvzalloc() */
636+ if (length > 16*1024*1024)
637+ return -ENOMEM;
638+
639+ if (flags & ~(MOCK_FLAGS_ACCESS_WRITE | MOCK_FLAGS_ACCESS_SYZ))
640+ return -EOPNOTSUPP;
641+
642+ staccess = iommufd_access_get(access_id);
643+ if (IS_ERR(staccess))
644+ return PTR_ERR(staccess);
645+
646+ if (staccess->access->ops != &selftest_access_ops_pin) {
647+ rc = -EOPNOTSUPP;
648+ goto out_put;
649+ }
650+
651+ if (flags & MOCK_FLAGS_ACCESS_SYZ)
652+ iova = iommufd_test_syz_conv_iova(&staccess->access->ioas->iopt,
653+ &cmd->access_pages.iova);
654+
655+ npages = (ALIGN(iova + length, PAGE_SIZE) -
656+ ALIGN_DOWN(iova, PAGE_SIZE)) /
657+ PAGE_SIZE;
658+ pages = kvcalloc(npages, sizeof(*pages), GFP_KERNEL_ACCOUNT);
659+ if (!pages) {
660+ rc = -ENOMEM;
661+ goto out_put;
662+ }
663+
664+ /*
665+ * Drivers will need to think very carefully about this locking. The
666+ * core code can do multiple unmaps instantaneously after
667+ * iommufd_access_pin_pages() and *all* the unmaps must not return until
668+ * the range is unpinned. This simple implementation puts a global lock
669+ * around the pin, which may not suit drivers that want this to be a
670+ * performance path. drivers that get this wrong will trigger WARN_ON
671+ * races and cause EDEADLOCK failures to userspace.
672+ */
673+ mutex_lock(&staccess->lock);
674+ rc = iommufd_access_pin_pages(staccess->access, iova, length, pages,
675+ flags & MOCK_FLAGS_ACCESS_WRITE);
676+ if (rc)
677+ goto out_unlock;
678+
679+ /* For syzkaller allow uptr to be NULL to skip this check */
680+ if (uptr) {
681+ rc = iommufd_test_check_pages(
682+ uptr - (iova - ALIGN_DOWN(iova, PAGE_SIZE)), pages,
683+ npages);
684+ if (rc)
685+ goto out_unaccess;
686+ }
687+
688+ item = kzalloc(sizeof(*item), GFP_KERNEL_ACCOUNT);
689+ if (!item) {
690+ rc = -ENOMEM;
691+ goto out_unaccess;
692+ }
693+
694+ item->iova = iova;
695+ item->length = length;
696+ item->id = staccess->next_id++;
697+ list_add_tail(&item->items_elm, &staccess->items);
698+
699+ cmd->access_pages.out_access_pages_id = item->id;
700+ rc = iommufd_ucmd_respond(ucmd, sizeof(*cmd));
701+ if (rc)
702+ goto out_free_item;
703+ goto out_unlock;
704+
705+out_free_item:
706+ list_del(&item->items_elm);
707+ kfree(item);
708+out_unaccess:
709+ iommufd_access_unpin_pages(staccess->access, iova, length);
710+out_unlock:
711+ mutex_unlock(&staccess->lock);
712+ kvfree(pages);
713+out_put:
714+ fput(staccess->file);
715+ return rc;
716+}
717+
718+static int iommufd_test_access_rw(struct iommufd_ucmd *ucmd,
719+ unsigned int access_id, unsigned long iova,
720+ size_t length, void __user *ubuf,
721+ unsigned int flags)
722+{
723+ struct iommu_test_cmd *cmd = ucmd->cmd;
724+ struct selftest_access *staccess;
725+ void *tmp;
726+ int rc;
727+
728+ /* Prevent syzkaller from triggering a WARN_ON in kvzalloc() */
729+ if (length > 16*1024*1024)
730+ return -ENOMEM;
731+
732+ if (flags & ~(MOCK_ACCESS_RW_WRITE | MOCK_ACCESS_RW_SLOW_PATH |
733+ MOCK_FLAGS_ACCESS_SYZ))
734+ return -EOPNOTSUPP;
735+
736+ staccess = iommufd_access_get(access_id);
737+ if (IS_ERR(staccess))
738+ return PTR_ERR(staccess);
739+
740+ tmp = kvzalloc(length, GFP_KERNEL_ACCOUNT);
741+ if (!tmp) {
742+ rc = -ENOMEM;
743+ goto out_put;
744+ }
745+
746+ if (flags & MOCK_ACCESS_RW_WRITE) {
747+ if (copy_from_user(tmp, ubuf, length)) {
748+ rc = -EFAULT;
749+ goto out_free;
750+ }
751+ }
752+
753+ if (flags & MOCK_FLAGS_ACCESS_SYZ)
754+ iova = iommufd_test_syz_conv_iova(&staccess->access->ioas->iopt,
755+ &cmd->access_rw.iova);
756+
757+ rc = iommufd_access_rw(staccess->access, iova, tmp, length, flags);
758+ if (rc)
759+ goto out_free;
760+ if (!(flags & MOCK_ACCESS_RW_WRITE)) {
761+ if (copy_to_user(ubuf, tmp, length)) {
762+ rc = -EFAULT;
763+ goto out_free;
764+ }
765+ }
766+
767+out_free:
768+ kvfree(tmp);
769+out_put:
770+ fput(staccess->file);
771+ return rc;
772+}
773+static_assert((unsigned int)MOCK_ACCESS_RW_WRITE == IOMMUFD_ACCESS_RW_WRITE);
774+static_assert((unsigned int)MOCK_ACCESS_RW_SLOW_PATH ==
775+ __IOMMUFD_ACCESS_RW_SLOW_PATH);
776+
777+void iommufd_selftest_destroy(struct iommufd_object *obj)
778+{
779+ struct selftest_obj *sobj = container_of(obj, struct selftest_obj, obj);
780+
781+ switch (sobj->type) {
782+ case TYPE_IDEV:
783+ iommufd_device_selftest_detach(sobj->idev.ictx,
784+ sobj->idev.hwpt);
785+ break;
786+ }
787+}
788+
789+int iommufd_test(struct iommufd_ucmd *ucmd)
790+{
791+ struct iommu_test_cmd *cmd = ucmd->cmd;
792+
793+ switch (cmd->op) {
794+ case IOMMU_TEST_OP_ADD_RESERVED:
795+ return iommufd_test_add_reserved(ucmd, cmd->id,
796+ cmd->add_reserved.start,
797+ cmd->add_reserved.length);
798+ case IOMMU_TEST_OP_MOCK_DOMAIN:
799+ return iommufd_test_mock_domain(ucmd, cmd);
800+ case IOMMU_TEST_OP_MD_CHECK_MAP:
801+ return iommufd_test_md_check_pa(
802+ ucmd, cmd->id, cmd->check_map.iova,
803+ cmd->check_map.length,
804+ u64_to_user_ptr(cmd->check_map.uptr));
805+ case IOMMU_TEST_OP_MD_CHECK_REFS:
806+ return iommufd_test_md_check_refs(
807+ ucmd, u64_to_user_ptr(cmd->check_refs.uptr),
808+ cmd->check_refs.length, cmd->check_refs.refs);
809+ case IOMMU_TEST_OP_CREATE_ACCESS:
810+ return iommufd_test_create_access(ucmd, cmd->id,
811+ cmd->create_access.flags);
812+ case IOMMU_TEST_OP_ACCESS_PAGES:
813+ return iommufd_test_access_pages(
814+ ucmd, cmd->id, cmd->access_pages.iova,
815+ cmd->access_pages.length,
816+ u64_to_user_ptr(cmd->access_pages.uptr),
817+ cmd->access_pages.flags);
818+ case IOMMU_TEST_OP_ACCESS_RW:
819+ return iommufd_test_access_rw(
820+ ucmd, cmd->id, cmd->access_rw.iova,
821+ cmd->access_rw.length,
822+ u64_to_user_ptr(cmd->access_rw.uptr),
823+ cmd->access_rw.flags);
824+ case IOMMU_TEST_OP_DESTROY_ACCESS_PAGES:
825+ return iommufd_test_access_item_destroy(
826+ ucmd, cmd->id, cmd->destroy_access_pages.access_pages_id);
827+ case IOMMU_TEST_OP_SET_TEMP_MEMORY_LIMIT:
828+ /* Protect _batch_init(), can not be less than elmsz */
829+ if (cmd->memory_limit.limit <
830+ sizeof(unsigned long) + sizeof(u32))
831+ return -EINVAL;
832+ iommufd_test_memory_limit = cmd->memory_limit.limit;
833+ return 0;
834+ default:
835+ return -EOPNOTSUPP;
836+ }
837+}
838+
839+bool iommufd_should_fail(void)
840+{
841+ return should_fail(&fail_iommufd, 1);
842+}
843+
844+void __init iommufd_test_init(void)
845+{
846+ dbgfs_root =
847+ fault_create_debugfs_attr("fail_iommufd", NULL, &fail_iommufd);
848+}
849+
850+void iommufd_test_exit(void)
851+{
852+ debugfs_remove_recursive(dbgfs_root);
853+}
drivers/iommu/iommufd/vfio_compat.c+472−0
@@ -0,0 +1,472 @@
1+// SPDX-License-Identifier: GPL-2.0-only
2+/* Copyright (c) 2021-2022, NVIDIA CORPORATION & AFFILIATES
3+ */
4+#include <linux/file.h>
5+#include <linux/interval_tree.h>
6+#include <linux/iommu.h>
7+#include <linux/iommufd.h>
8+#include <linux/slab.h>
9+#include <linux/vfio.h>
10+#include <uapi/linux/vfio.h>
11+#include <uapi/linux/iommufd.h>
12+
13+#include "iommufd_private.h"
14+
15+static struct iommufd_ioas *get_compat_ioas(struct iommufd_ctx *ictx)
16+{
17+ struct iommufd_ioas *ioas = ERR_PTR(-ENODEV);
18+
19+ xa_lock(&ictx->objects);
20+ if (!ictx->vfio_ioas || !iommufd_lock_obj(&ictx->vfio_ioas->obj))
21+ goto out_unlock;
22+ ioas = ictx->vfio_ioas;
23+out_unlock:
24+ xa_unlock(&ictx->objects);
25+ return ioas;
26+}
27+
28+/**
29+ * iommufd_vfio_compat_ioas_id - Return the IOAS ID that vfio should use
30+ * @ictx: Context to operate on
31+ * @out_ioas_id: The ioas_id the caller should use
32+ *
33+ * The compatibility IOAS is the IOAS that the vfio compatibility ioctls operate
34+ * on since they do not have an IOAS ID input in their ABI. Only attaching a
35+ * group should cause a default creation of the internal ioas, this returns the
36+ * existing ioas if it has already been assigned somehow.
37+ */
38+int iommufd_vfio_compat_ioas_id(struct iommufd_ctx *ictx, u32 *out_ioas_id)
39+{
40+ struct iommufd_ioas *ioas = NULL;
41+ struct iommufd_ioas *out_ioas;
42+
43+ ioas = iommufd_ioas_alloc(ictx);
44+ if (IS_ERR(ioas))
45+ return PTR_ERR(ioas);
46+
47+ xa_lock(&ictx->objects);
48+ if (ictx->vfio_ioas && iommufd_lock_obj(&ictx->vfio_ioas->obj))
49+ out_ioas = ictx->vfio_ioas;
50+ else {
51+ out_ioas = ioas;
52+ ictx->vfio_ioas = ioas;
53+ }
54+ xa_unlock(&ictx->objects);
55+
56+ *out_ioas_id = out_ioas->obj.id;
57+ if (out_ioas != ioas) {
58+ iommufd_put_object(&out_ioas->obj);
59+ iommufd_object_abort(ictx, &ioas->obj);
60+ return 0;
61+ }
62+ /*
63+ * An automatically created compat IOAS is treated as a userspace
64+ * created object. Userspace can learn the ID via IOMMU_VFIO_IOAS_GET,
65+ * and if not manually destroyed it will be destroyed automatically
66+ * at iommufd release.
67+ */
68+ iommufd_object_finalize(ictx, &ioas->obj);
69+ return 0;
70+}
71+EXPORT_SYMBOL_NS_GPL(iommufd_vfio_compat_ioas_id, IOMMUFD_VFIO);
72+
73+int iommufd_vfio_ioas(struct iommufd_ucmd *ucmd)
74+{
75+ struct iommu_vfio_ioas *cmd = ucmd->cmd;
76+ struct iommufd_ioas *ioas;
77+
78+ if (cmd->__reserved)
79+ return -EOPNOTSUPP;
80+ switch (cmd->op) {
81+ case IOMMU_VFIO_IOAS_GET:
82+ ioas = get_compat_ioas(ucmd->ictx);
83+ if (IS_ERR(ioas))
84+ return PTR_ERR(ioas);
85+ cmd->ioas_id = ioas->obj.id;
86+ iommufd_put_object(&ioas->obj);
87+ return iommufd_ucmd_respond(ucmd, sizeof(*cmd));
88+
89+ case IOMMU_VFIO_IOAS_SET:
90+ ioas = iommufd_get_ioas(ucmd, cmd->ioas_id);
91+ if (IS_ERR(ioas))
92+ return PTR_ERR(ioas);
93+ xa_lock(&ucmd->ictx->objects);
94+ ucmd->ictx->vfio_ioas = ioas;
95+ xa_unlock(&ucmd->ictx->objects);
96+ iommufd_put_object(&ioas->obj);
97+ return 0;
98+
99+ case IOMMU_VFIO_IOAS_CLEAR:
100+ xa_lock(&ucmd->ictx->objects);
101+ ucmd->ictx->vfio_ioas = NULL;
102+ xa_unlock(&ucmd->ictx->objects);
103+ return 0;
104+ default:
105+ return -EOPNOTSUPP;
106+ }
107+}
108+
109+static int iommufd_vfio_map_dma(struct iommufd_ctx *ictx, unsigned int cmd,
110+ void __user *arg)
111+{
112+ u32 supported_flags = VFIO_DMA_MAP_FLAG_READ | VFIO_DMA_MAP_FLAG_WRITE;
113+ size_t minsz = offsetofend(struct vfio_iommu_type1_dma_map, size);
114+ struct vfio_iommu_type1_dma_map map;
115+ int iommu_prot = IOMMU_CACHE;
116+ struct iommufd_ioas *ioas;
117+ unsigned long iova;
118+ int rc;
119+
120+ if (copy_from_user(&map, arg, minsz))
121+ return -EFAULT;
122+
123+ if (map.argsz < minsz || map.flags & ~supported_flags)
124+ return -EINVAL;
125+
126+ if (map.flags & VFIO_DMA_MAP_FLAG_READ)
127+ iommu_prot |= IOMMU_READ;
128+ if (map.flags & VFIO_DMA_MAP_FLAG_WRITE)
129+ iommu_prot |= IOMMU_WRITE;
130+
131+ ioas = get_compat_ioas(ictx);
132+ if (IS_ERR(ioas))
133+ return PTR_ERR(ioas);
134+
135+ /*
136+ * Maps created through the legacy interface always use VFIO compatible
137+ * rlimit accounting. If the user wishes to use the faster user based
138+ * rlimit accounting then they must use the new interface.
139+ */
140+ iova = map.iova;
141+ rc = iopt_map_user_pages(ictx, &ioas->iopt, &iova, u64_to_user_ptr(map.vaddr),
142+ map.size, iommu_prot, 0);
143+ iommufd_put_object(&ioas->obj);
144+ return rc;
145+}
146+
147+static int iommufd_vfio_unmap_dma(struct iommufd_ctx *ictx, unsigned int cmd,
148+ void __user *arg)
149+{
150+ size_t minsz = offsetofend(struct vfio_iommu_type1_dma_unmap, size);
151+ /*
152+ * VFIO_DMA_UNMAP_FLAG_GET_DIRTY_BITMAP is obsoleted by the new
153+ * dirty tracking direction:
154+ * https://lore.kernel.org/kvm/20220731125503.142683-1-yishaih@nvidia.com/
155+ * https://lore.kernel.org/kvm/20220428210933.3583-1-joao.m.martins@oracle.com/
156+ */
157+ u32 supported_flags = VFIO_DMA_UNMAP_FLAG_ALL;
158+ struct vfio_iommu_type1_dma_unmap unmap;
159+ unsigned long unmapped = 0;
160+ struct iommufd_ioas *ioas;
161+ int rc;
162+
163+ if (copy_from_user(&unmap, arg, minsz))
164+ return -EFAULT;
165+
166+ if (unmap.argsz < minsz || unmap.flags & ~supported_flags)
167+ return -EINVAL;
168+
169+ ioas = get_compat_ioas(ictx);
170+ if (IS_ERR(ioas))
171+ return PTR_ERR(ioas);
172+
173+ if (unmap.flags & VFIO_DMA_UNMAP_FLAG_ALL) {
174+ if (unmap.iova != 0 || unmap.size != 0) {
175+ rc = -EINVAL;
176+ goto err_put;
177+ }
178+ rc = iopt_unmap_all(&ioas->iopt, &unmapped);
179+ } else {
180+ if (READ_ONCE(ioas->iopt.disable_large_pages)) {
181+ /*
182+ * Create cuts at the start and last of the requested
183+ * range. If the start IOVA is 0 then it doesn't need to
184+ * be cut.
185+ */
186+ unsigned long iovas[] = { unmap.iova + unmap.size - 1,
187+ unmap.iova - 1 };
188+
189+ rc = iopt_cut_iova(&ioas->iopt, iovas,
190+ unmap.iova ? 2 : 1);
191+ if (rc)
192+ goto err_put;
193+ }
194+ rc = iopt_unmap_iova(&ioas->iopt, unmap.iova, unmap.size,
195+ &unmapped);
196+ }
197+ unmap.size = unmapped;
198+ if (copy_to_user(arg, &unmap, minsz))
199+ rc = -EFAULT;
200+
201+err_put:
202+ iommufd_put_object(&ioas->obj);
203+ return rc;
204+}
205+
206+static int iommufd_vfio_cc_iommu(struct iommufd_ctx *ictx)
207+{
208+ struct iommufd_hw_pagetable *hwpt;
209+ struct iommufd_ioas *ioas;
210+ int rc = 1;
211+
212+ ioas = get_compat_ioas(ictx);
213+ if (IS_ERR(ioas))
214+ return PTR_ERR(ioas);
215+
216+ mutex_lock(&ioas->mutex);
217+ list_for_each_entry(hwpt, &ioas->hwpt_list, hwpt_item) {
218+ if (!hwpt->enforce_cache_coherency) {
219+ rc = 0;
220+ break;
221+ }
222+ }
223+ mutex_unlock(&ioas->mutex);
224+
225+ iommufd_put_object(&ioas->obj);
226+ return rc;
227+}
228+
229+static int iommufd_vfio_check_extension(struct iommufd_ctx *ictx,
230+ unsigned long type)
231+{
232+ switch (type) {
233+ case VFIO_TYPE1_IOMMU:
234+ case VFIO_TYPE1v2_IOMMU:
235+ case VFIO_UNMAP_ALL:
236+ return 1;
237+
238+ case VFIO_DMA_CC_IOMMU:
239+ return iommufd_vfio_cc_iommu(ictx);
240+
241+ /*
242+ * This is obsolete, and to be removed from VFIO. It was an incomplete
243+ * idea that got merged.
244+ * https://lore.kernel.org/kvm/0-v1-0093c9b0e345+19-vfio_no_nesting_jgg@nvidia.com/
245+ */
246+ case VFIO_TYPE1_NESTING_IOMMU:
247+ return 0;
248+
249+ /*
250+ * VFIO_DMA_MAP_FLAG_VADDR
251+ * https://lore.kernel.org/kvm/1611939252-7240-1-git-send-email-steven.sistare@oracle.com/
252+ * https://lore.kernel.org/all/Yz777bJZjTyLrHEQ@nvidia.com/
253+ *
254+ * It is hard to see how this could be implemented safely.
255+ */
256+ case VFIO_UPDATE_VADDR:
257+ default:
258+ return 0;
259+ }
260+}
261+
262+static int iommufd_vfio_set_iommu(struct iommufd_ctx *ictx, unsigned long type)
263+{
264+ struct iommufd_ioas *ioas = NULL;
265+ int rc = 0;
266+
267+ if (type != VFIO_TYPE1_IOMMU && type != VFIO_TYPE1v2_IOMMU)
268+ return -EINVAL;
269+
270+ /* VFIO fails the set_iommu if there is no group */
271+ ioas = get_compat_ioas(ictx);
272+ if (IS_ERR(ioas))
273+ return PTR_ERR(ioas);
274+
275+ /*
276+ * The difference between TYPE1 and TYPE1v2 is the ability to unmap in
277+ * the middle of mapped ranges. This is complicated by huge page support
278+ * which creates single large IOPTEs that cannot be split by the iommu
279+ * driver. TYPE1 is very old at this point and likely nothing uses it,
280+ * however it is simple enough to emulate by simply disabling the
281+ * problematic large IOPTEs. Then we can safely unmap within any range.
282+ */
283+ if (type == VFIO_TYPE1_IOMMU)
284+ rc = iopt_disable_large_pages(&ioas->iopt);
285+ iommufd_put_object(&ioas->obj);
286+ return rc;
287+}
288+
289+static unsigned long iommufd_get_pagesizes(struct iommufd_ioas *ioas)
290+{
291+ struct io_pagetable *iopt = &ioas->iopt;
292+ unsigned long pgsize_bitmap = ULONG_MAX;
293+ struct iommu_domain *domain;
294+ unsigned long index;
295+
296+ down_read(&iopt->domains_rwsem);
297+ xa_for_each(&iopt->domains, index, domain)
298+ pgsize_bitmap &= domain->pgsize_bitmap;
299+
300+ /* See vfio_update_pgsize_bitmap() */
301+ if (pgsize_bitmap & ~PAGE_MASK) {
302+ pgsize_bitmap &= PAGE_MASK;
303+ pgsize_bitmap |= PAGE_SIZE;
304+ }
305+ pgsize_bitmap = max(pgsize_bitmap, ioas->iopt.iova_alignment);
306+ up_read(&iopt->domains_rwsem);
307+ return pgsize_bitmap;
308+}
309+
310+static int iommufd_fill_cap_iova(struct iommufd_ioas *ioas,
311+ struct vfio_info_cap_header __user *cur,
312+ size_t avail)
313+{
314+ struct vfio_iommu_type1_info_cap_iova_range __user *ucap_iovas =
315+ container_of(cur,
316+ struct vfio_iommu_type1_info_cap_iova_range __user,
317+ header);
318+ struct vfio_iommu_type1_info_cap_iova_range cap_iovas = {
319+ .header = {
320+ .id = VFIO_IOMMU_TYPE1_INFO_CAP_IOVA_RANGE,
321+ .version = 1,
322+ },
323+ };
324+ struct interval_tree_span_iter span;
325+
326+ interval_tree_for_each_span(&span, &ioas->iopt.reserved_itree, 0,
327+ ULONG_MAX) {
328+ struct vfio_iova_range range;
329+
330+ if (!span.is_hole)
331+ continue;
332+ range.start = span.start_hole;
333+ range.end = span.last_hole;
334+ if (avail >= struct_size(&cap_iovas, iova_ranges,
335+ cap_iovas.nr_iovas + 1) &&
336+ copy_to_user(&ucap_iovas->iova_ranges[cap_iovas.nr_iovas],
337+ &range, sizeof(range)))
338+ return -EFAULT;
339+ cap_iovas.nr_iovas++;
340+ }
341+ if (avail >= struct_size(&cap_iovas, iova_ranges, cap_iovas.nr_iovas) &&
342+ copy_to_user(ucap_iovas, &cap_iovas, sizeof(cap_iovas)))
343+ return -EFAULT;
344+ return struct_size(&cap_iovas, iova_ranges, cap_iovas.nr_iovas);
345+}
346+
347+static int iommufd_fill_cap_dma_avail(struct iommufd_ioas *ioas,
348+ struct vfio_info_cap_header __user *cur,
349+ size_t avail)
350+{
351+ struct vfio_iommu_type1_info_dma_avail cap_dma = {
352+ .header = {
353+ .id = VFIO_IOMMU_TYPE1_INFO_DMA_AVAIL,
354+ .version = 1,
355+ },
356+ /*
357+ * iommufd's limit is based on the cgroup's memory limit.
358+ * Normally vfio would return U16_MAX here, and provide a module
359+ * parameter to adjust it. Since S390 qemu userspace actually
360+ * pays attention and needs a value bigger than U16_MAX return
361+ * U32_MAX.
362+ */
363+ .avail = U32_MAX,
364+ };
365+
366+ if (avail >= sizeof(cap_dma) &&
367+ copy_to_user(cur, &cap_dma, sizeof(cap_dma)))
368+ return -EFAULT;
369+ return sizeof(cap_dma);
370+}
371+
372+static int iommufd_vfio_iommu_get_info(struct iommufd_ctx *ictx,
373+ void __user *arg)
374+{
375+ typedef int (*fill_cap_fn)(struct iommufd_ioas *ioas,
376+ struct vfio_info_cap_header __user *cur,
377+ size_t avail);
378+ static const fill_cap_fn fill_fns[] = {
379+ iommufd_fill_cap_dma_avail,
380+ iommufd_fill_cap_iova,
381+ };
382+ size_t minsz = offsetofend(struct vfio_iommu_type1_info, iova_pgsizes);
383+ struct vfio_info_cap_header __user *last_cap = NULL;
384+ struct vfio_iommu_type1_info info;
385+ struct iommufd_ioas *ioas;
386+ size_t total_cap_size;
387+ int rc;
388+ int i;
389+
390+ if (copy_from_user(&info, arg, minsz))
391+ return -EFAULT;
392+
393+ if (info.argsz < minsz)
394+ return -EINVAL;
395+ minsz = min_t(size_t, info.argsz, sizeof(info));
396+
397+ ioas = get_compat_ioas(ictx);
398+ if (IS_ERR(ioas))
399+ return PTR_ERR(ioas);
400+
401+ info.flags = VFIO_IOMMU_INFO_PGSIZES;
402+ info.iova_pgsizes = iommufd_get_pagesizes(ioas);
403+ info.cap_offset = 0;
404+
405+ down_read(&ioas->iopt.iova_rwsem);
406+ total_cap_size = sizeof(info);
407+ for (i = 0; i != ARRAY_SIZE(fill_fns); i++) {
408+ int cap_size;
409+
410+ if (info.argsz > total_cap_size)
411+ cap_size = fill_fns[i](ioas, arg + total_cap_size,
412+ info.argsz - total_cap_size);
413+ else
414+ cap_size = fill_fns[i](ioas, NULL, 0);
415+ if (cap_size < 0) {
416+ rc = cap_size;
417+ goto out_put;
418+ }
419+ if (last_cap && info.argsz >= total_cap_size &&
420+ put_user(total_cap_size, &last_cap->next)) {
421+ rc = -EFAULT;
422+ goto out_put;
423+ }
424+ last_cap = arg + total_cap_size;
425+ total_cap_size += cap_size;
426+ }
427+
428+ /*
429+ * If the user did not provide enough space then only some caps are
430+ * returned and the argsz will be updated to the correct amount to get
431+ * all caps.
432+ */
433+ if (info.argsz >= total_cap_size)
434+ info.cap_offset = sizeof(info);
435+ info.argsz = total_cap_size;
436+ info.flags |= VFIO_IOMMU_INFO_CAPS;
437+ if (copy_to_user(arg, &info, minsz)) {
438+ rc = -EFAULT;
439+ goto out_put;
440+ }
441+ rc = 0;
442+
443+out_put:
444+ up_read(&ioas->iopt.iova_rwsem);
445+ iommufd_put_object(&ioas->obj);
446+ return rc;
447+}
448+
449+int iommufd_vfio_ioctl(struct iommufd_ctx *ictx, unsigned int cmd,
450+ unsigned long arg)
451+{
452+ void __user *uarg = (void __user *)arg;
453+
454+ switch (cmd) {
455+ case VFIO_GET_API_VERSION:
456+ return VFIO_API_VERSION;
457+ case VFIO_SET_IOMMU:
458+ return iommufd_vfio_set_iommu(ictx, arg);
459+ case VFIO_CHECK_EXTENSION:
460+ return iommufd_vfio_check_extension(ictx, arg);
461+ case VFIO_IOMMU_GET_INFO:
462+ return iommufd_vfio_iommu_get_info(ictx, uarg);
463+ case VFIO_IOMMU_MAP_DMA:
464+ return iommufd_vfio_map_dma(ictx, cmd, uarg);
465+ case VFIO_IOMMU_UNMAP_DMA:
466+ return iommufd_vfio_unmap_dma(ictx, cmd, uarg);
467+ case VFIO_IOMMU_DIRTY_PAGES:
468+ default:
469+ return -ENOIOCTLCMD;
470+ }
471+ return -ENOIOCTLCMD;
472+}
drivers/iommu/ipmmu-vmsa.c+0−2
@@ -628,8 +628,6 @@ static int ipmmu_attach_device(struct iommu_domain *io_domain, · 627 unmodified lines
628628 * Something is wrong, we can't attach two devices using
629629 * different IOMMUs to the same domain.
630630 */
631− dev_err(dev, "Can't attach IPMMU %s to domain on IPMMU %s\n",
632− dev_name(mmu->dev), dev_name(domain->mmu->dev));
633631 ret = -EINVAL;
634632 } else
635633 dev_info(dev, "Reusing IPMMU context %u\n", domain->context_id);
drivers/iommu/mtk_iommu.c+2−2
@@ -609,7 +609,7 @@ static int mtk_iommu_domain_finalise(struct mtk_iommu_domain *dom, · 608 unmodified lines
609609 dom->iop = alloc_io_pgtable_ops(ARM_V7S, &dom->cfg, data);
610610 if (!dom->iop) {
611611 dev_err(data->dev, "Failed to alloc io pgtable\n");
612− return -EINVAL;
612+ return -ENOMEM;
613613 }
614614
615615 /* Update our support page sizes bitmap */
@@ -668,7 +668,7 @@ static int mtk_iommu_attach_device(struct iommu_domain *domain, · 52 unmodified lines
668668 ret = mtk_iommu_domain_finalise(dom, frstdata, region_id);
669669 if (ret) {
670670 mutex_unlock(&dom->mutex);
671− return -ENODEV;
671+ return ret;
672672 }
673673 dom->bank = &data->bank[bankid];
674674 }
drivers/iommu/omap-iommu.c+3−3
@@ -1414,7 +1414,7 @@ static int omap_iommu_attach_init(struct device *dev, · 1413 unmodified lines
14141414
14151415 odomain->num_iommus = omap_iommu_count(dev);
14161416 if (!odomain->num_iommus)
1417− return -EINVAL;
1417+ return -ENODEV;
14181418
14191419 odomain->iommus = kcalloc(odomain->num_iommus, sizeof(*iommu),
14201420 GFP_ATOMIC);
@@ -1464,7 +1464,7 @@ omap_iommu_attach_dev(struct iommu_domain *domain, struct device *dev) · 43 unmodified lines
14641464
14651465 if (!arch_data || !arch_data->iommu_dev) {
14661466 dev_err(dev, "device doesn't have an associated iommu\n");
1467− return -EINVAL;
1467+ return -ENODEV;
14681468 }
14691469
14701470 spin_lock(&omap_domain->lock);
@@ -1472,7 +1472,7 @@ omap_iommu_attach_dev(struct iommu_domain *domain, struct device *dev) · 1 unmodified line
14721472 /* only a single client device can be attached to a domain */
14731473 if (omap_domain->dev) {
14741474 dev_err(dev, "iommu domain is already attached\n");
1475− ret = -EBUSY;
1475+ ret = -EINVAL;
14761476 goto out;
14771477 }
14781478
drivers/iommu/sprd-iommu.c+1−3
@@ -237,10 +237,8 @@ static int sprd_iommu_attach_device(struct iommu_domain *domain, · 236 unmodified lines
237237 struct sprd_iommu_domain *dom = to_sprd_domain(domain);
238238 size_t pgt_size = sprd_iommu_pgt_size(domain);
239239
240− if (dom->sdev) {
241− pr_err("There's already a device attached to this domain.\n");
240+ if (dom->sdev)
242241 return -EINVAL;
243− }
244242
245243 dom->pgt_va = dma_alloc_coherent(sdev->dev, pgt_size, &dom->pgt_pa, GFP_KERNEL);
246244 if (!dom->pgt_va)
drivers/iommu/tegra-gart.c+1−1
@@ -112,7 +112,7 @@ static int gart_iommu_attach_dev(struct iommu_domain *domain, · 111 unmodified lines
112112 spin_lock(&gart->dom_lock);
113113
114114 if (gart->active_domain && gart->active_domain != domain) {
115− ret = -EBUSY;
115+ ret = -EINVAL;
116116 } else if (dev_iommu_priv_get(dev) != domain) {
117117 dev_iommu_priv_set(dev, domain);
118118 gart->active_domain = domain;
drivers/iommu/virtio-iommu.c+3−4
@@ -670,7 +670,7 @@ static int viommu_domain_finalise(struct viommu_endpoint *vdev, · 669 unmodified lines
670670 dev_err(vdev->dev,
671671 "granule 0x%lx larger than system page size 0x%lx\n",
672672 viommu_page_size, PAGE_SIZE);
673− return -EINVAL;
673+ return -ENODEV;
674674 }
675675
676676 ret = ida_alloc_range(&viommu->domain_ids, viommu->first_domain,
@@ -697,7 +697,7 @@ static int viommu_domain_finalise(struct viommu_endpoint *vdev, · 20 unmodified lines
697697 if (ret) {
698698 ida_free(&viommu->domain_ids, vdomain->id);
699699 vdomain->viommu = NULL;
700− return -EOPNOTSUPP;
700+ return ret;
701701 }
702702 }
703703
@@ -734,8 +734,7 @@ static int viommu_attach_dev(struct iommu_domain *domain, struct device *dev) · 30 unmodified lines
734734 */
735735 ret = viommu_domain_finalise(vdev, domain);
736736 } else if (vdomain->viommu != vdev->viommu) {
737− dev_err(dev, "cannot attach to foreign vIOMMU\n");
738− ret = -EXDEV;
737+ ret = -EINVAL;
739738 }
740739 mutex_unlock(&vdomain->mutex);
741740
drivers/misc/uacce/uacce.c+1−1
@@ -108,7 +108,7 @@ static int uacce_bind_queue(struct uacce_device *uacce, struct uacce_queue *q) · 107 unmodified lines
108108 if (!(uacce->flags & UACCE_DEV_SVA))
109109 return 0;
110110
111− handle = iommu_sva_bind_device(uacce->parent, current->mm, NULL);
111+ handle = iommu_sva_bind_device(uacce->parent, current->mm);
112112 if (IS_ERR(handle))
113113 return PTR_ERR(handle);
114114
drivers/pci/ats.c+3−0
@@ -382,6 +382,9 @@ int pci_enable_pasid(struct pci_dev *pdev, int features) · 381 unmodified lines
382382 if (!pasid)
383383 return -EINVAL;
384384
385+ if (!pci_acs_path_enabled(pdev, NULL, PCI_ACS_RR | PCI_ACS_UF))
386+ return -EINVAL;
387+
385388 pci_read_config_word(pdev, pasid + PCI_PASID_CAP, &supported);
386389 supported &= PCI_PASID_CAP_EXEC | PCI_PASID_CAP_PRIV;
387390
drivers/s390/cio/vfio_ccw_ops.c+3−0
@@ -588,6 +588,9 @@ static const struct vfio_device_ops vfio_ccw_dev_ops = { · 587 unmodified lines
588588 .ioctl = vfio_ccw_mdev_ioctl,
589589 .request = vfio_ccw_mdev_request,
590590 .dma_unmap = vfio_ccw_dma_unmap,
591+ .bind_iommufd = vfio_iommufd_emulated_bind,
592+ .unbind_iommufd = vfio_iommufd_emulated_unbind,
593+ .attach_ioas = vfio_iommufd_emulated_attach_ioas,
591594 };
592595
593596 struct mdev_driver vfio_ccw_mdev_driver = {
drivers/s390/crypto/vfio_ap_ops.c+20−1
@@ -1535,13 +1535,29 @@ static int vfio_ap_mdev_set_kvm(struct ap_matrix_mdev *matrix_mdev, · 1534 unmodified lines
15351535 return 0;
15361536 }
15371537
1538+static void unmap_iova(struct ap_matrix_mdev *matrix_mdev, u64 iova, u64 length)
1539+{
1540+ struct ap_queue_table *qtable = &matrix_mdev->qtable;
1541+ struct vfio_ap_queue *q;
1542+ int loop_cursor;
1543+
1544+ hash_for_each(qtable->queues, loop_cursor, q, mdev_qnode) {
1545+ if (q->saved_iova >= iova && q->saved_iova < iova + length)
1546+ vfio_ap_irq_disable(q);
1547+ }
1548+}
1549+
15381550 static void vfio_ap_mdev_dma_unmap(struct vfio_device *vdev, u64 iova,
15391551 u64 length)
15401552 {
15411553 struct ap_matrix_mdev *matrix_mdev =
15421554 container_of(vdev, struct ap_matrix_mdev, vdev);
15431555
1544− vfio_unpin_pages(&matrix_mdev->vdev, iova, 1);
1556+ mutex_lock(&matrix_dev->mdevs_lock);
1557+
1558+ unmap_iova(matrix_mdev, iova, length);
1559+
1560+ mutex_unlock(&matrix_dev->mdevs_lock);
15451561 }
15461562
15471563 /**
@@ -1789,6 +1805,9 @@ static const struct vfio_device_ops vfio_ap_matrix_dev_ops = { · 241 unmodified lines
17891805 .close_device = vfio_ap_mdev_close_device,
17901806 .ioctl = vfio_ap_mdev_ioctl,
17911807 .dma_unmap = vfio_ap_mdev_dma_unmap,
1808+ .bind_iommufd = vfio_iommufd_emulated_bind,
1809+ .unbind_iommufd = vfio_iommufd_emulated_unbind,
1810+ .attach_ioas = vfio_iommufd_emulated_attach_ioas,
17921811 };
17931812
17941813 static struct mdev_driver vfio_ap_matrix_driver = {
drivers/vfio/Kconfig+25−11
@@ -2,8 +2,9 @@ · 1 unmodified line
22 menuconfig VFIO
33 tristate "VFIO Non-Privileged userspace driver framework"
44 select IOMMU_API
5− select VFIO_IOMMU_TYPE1 if MMU && (X86 || S390 || ARM || ARM64)
5+ depends on IOMMUFD || !IOMMUFD
66 select INTERVAL_TREE
7+ select VFIO_CONTAINER if IOMMUFD=n
78 help
89 VFIO provides a framework for secure userspace device drivers.
910 See Documentation/driver-api/vfio.rst for more details.
@@ -11,6 +12,18 @@ menuconfig VFIO · 1 unmodified line
1112 If you don't know what to do here, say N.
1213
1314 if VFIO
15+config VFIO_CONTAINER
16+ bool "Support for the VFIO container /dev/vfio/vfio"
17+ select VFIO_IOMMU_TYPE1 if MMU && (X86 || S390 || ARM || ARM64)
18+ default y
19+ help
20+ The VFIO container is the classic interface to VFIO for establishing
21+ IOMMU mappings. If N is selected here then IOMMUFD must be used to
22+ manage the mappings.
23+
24+ Unless testing IOMMUFD say Y here.
25+
26+if VFIO_CONTAINER
1427 config VFIO_IOMMU_TYPE1
1528 tristate
1629 default n
@@ -20,16 +33,6 @@ config VFIO_IOMMU_SPAPR_TCE · 3 unmodified lines
2033 depends on SPAPR_TCE_IOMMU
2134 default VFIO
2235
23−config VFIO_SPAPR_EEH
24− tristate
25− depends on EEH && VFIO_IOMMU_SPAPR_TCE
26− default VFIO
27−
28−config VFIO_VIRQFD
29− tristate
30− select EVENTFD
31− default n
32−
3336 config VFIO_NOIOMMU
3437 bool "VFIO No-IOMMU support"
3538 help
@@ -43,6 +46,17 @@ config VFIO_NOIOMMU · 7 unmodified lines
4346 this mode since there is no IOMMU to provide DMA translation.
4447
4548 If you don't know what to do here, say N.
49+endif
50+
51+config VFIO_SPAPR_EEH
52+ tristate
53+ depends on EEH && VFIO_IOMMU_SPAPR_TCE
54+ default VFIO
55+
56+config VFIO_VIRQFD
57+ tristate
58+ select EVENTFD
59+ default n
4660
4761 source "drivers/vfio/pci/Kconfig"
4862 source "drivers/vfio/platform/Kconfig"
drivers/vfio/Makefile+4−2
@@ -4,8 +4,10 @@ vfio_virqfd-y := virqfd.o · 3 unmodified lines
44 obj-$(CONFIG_VFIO) += vfio.o
55
66 vfio-y += vfio_main.o \
7− iova_bitmap.o \
8− container.o
7+ group.o \
8+ iova_bitmap.o
9+vfio-$(CONFIG_IOMMUFD) += iommufd.o
10+vfio-$(CONFIG_VFIO_CONTAINER) += container.o
911
1012 obj-$(CONFIG_VFIO_VIRQFD) += vfio_virqfd.o
1113 obj-$(CONFIG_VFIO_IOMMU_TYPE1) += vfio_iommu_type1.o
drivers/vfio/container.c+42−103
@@ -188,8 +188,9 @@ void vfio_device_container_unregister(struct vfio_device *device) · 187 unmodified lines
188188 device->group->container->iommu_data, device);
189189 }
190190
191−long vfio_container_ioctl_check_extension(struct vfio_container *container,
192− unsigned long arg)
191+static long
192+vfio_container_ioctl_check_extension(struct vfio_container *container,
193+ unsigned long arg)
193194 {
194195 struct vfio_iommu_driver *driver;
195196 long ret = 0;
@@ -511,14 +512,15 @@ void vfio_group_detach_container(struct vfio_group *group) · 315 unmodified lines
511512 vfio_container_put(container);
512513 }
513514
514−int vfio_device_assign_container(struct vfio_device *device)
515+int vfio_group_use_container(struct vfio_group *group)
515516 {
516− struct vfio_group *group = device->group;
517−
518517 lockdep_assert_held(&group->group_lock);
519518
520− if (!group->container || !group->container->iommu_driver ||
521− WARN_ON(!group->container_users))
519+ /*
520+ * The container fd has been assigned with VFIO_GROUP_SET_CONTAINER but
521+ * VFIO_SET_IOMMU hasn't been done yet.
522+ */
523+ if (!group->container->iommu_driver)
522524 return -EINVAL;
523525
524526 if (group->type == VFIO_NO_IOMMU && !capable(CAP_SYS_RAWIO))
@@ -529,122 +531,56 @@ int vfio_device_assign_container(struct vfio_device *device) · 4 unmodified lines
529531 return 0;
530532 }
531533
532−void vfio_device_unassign_container(struct vfio_device *device)
534+void vfio_group_unuse_container(struct vfio_group *group)
533535 {
534− mutex_lock(&device->group->group_lock);
535− WARN_ON(device->group->container_users <= 1);
536− device->group->container_users--;
537− fput(device->group->opened_file);
538− mutex_unlock(&device->group->group_lock);
536+ lockdep_assert_held(&group->group_lock);
537+
538+ WARN_ON(group->container_users <= 1);
539+ group->container_users--;
540+ fput(group->opened_file);
539541 }
540542
541−/*
542− * Pin contiguous user pages and return their associated host pages for local
543− * domain only.
544− * @device [in] : device
545− * @iova [in] : starting IOVA of user pages to be pinned.
546− * @npage [in] : count of pages to be pinned. This count should not
547− * be greater than VFIO_PIN_PAGES_MAX_ENTRIES.
548− * @prot [in] : protection flags
549− * @pages[out] : array of host pages
550− * Return error or number of pages pinned.
551− *
552− * A driver may only call this function if the vfio_device was created
553− * by vfio_register_emulated_iommu_dev().
554− */
555−int vfio_pin_pages(struct vfio_device *device, dma_addr_t iova,
556− int npage, int prot, struct page **pages)
543+int vfio_device_container_pin_pages(struct vfio_device *device,
544+ dma_addr_t iova, int npage,
545+ int prot, struct page **pages)
557546 {
558− struct vfio_container *container;
559− struct vfio_group *group = device->group;
560− struct vfio_iommu_driver *driver;
561− int ret;
562−
563− if (!pages || !npage || !vfio_assert_device_open(device))
564− return -EINVAL;
547+ struct vfio_container *container = device->group->container;
548+ struct iommu_group *iommu_group = device->group->iommu_group;
549+ struct vfio_iommu_driver *driver = container->iommu_driver;
565550
566551 if (npage > VFIO_PIN_PAGES_MAX_ENTRIES)
567552 return -E2BIG;
568553
569− /* group->container cannot change while a vfio device is open */
570− container = group->container;
571− driver = container->iommu_driver;
572− if (likely(driver && driver->ops->pin_pages))
573− ret = driver->ops->pin_pages(container->iommu_data,
574− group->iommu_group, iova,
575− npage, prot, pages);
576− else
577− ret = -ENOTTY;
578−
579− return ret;
554+ if (unlikely(!driver || !driver->ops->pin_pages))
555+ return -ENOTTY;
556+ return driver->ops->pin_pages(container->iommu_data, iommu_group, iova,
557+ npage, prot, pages);
580558 }
581−EXPORT_SYMBOL(vfio_pin_pages);
582559
583−/*
584− * Unpin contiguous host pages for local domain only.
585− * @device [in] : device
586− * @iova [in] : starting address of user pages to be unpinned.
587− * @npage [in] : count of pages to be unpinned. This count should not
588− * be greater than VFIO_PIN_PAGES_MAX_ENTRIES.
589− */
590−void vfio_unpin_pages(struct vfio_device *device, dma_addr_t iova, int npage)
560+void vfio_device_container_unpin_pages(struct vfio_device *device,
561+ dma_addr_t iova, int npage)
591562 {
592− struct vfio_container *container;
593− struct vfio_iommu_driver *driver;
563+ struct vfio_container *container = device->group->container;
594564
595565 if (WARN_ON(npage <= 0 || npage > VFIO_PIN_PAGES_MAX_ENTRIES))
596566 return;
597567
598− if (WARN_ON(!vfio_assert_device_open(device)))
599− return;
600−
601− /* group->container cannot change while a vfio device is open */
602− container = device->group->container;
603− driver = container->iommu_driver;
604−
605− driver->ops->unpin_pages(container->iommu_data, iova, npage);
568+ container->iommu_driver->ops->unpin_pages(container->iommu_data, iova,
569+ npage);
606570 }
607−EXPORT_SYMBOL(vfio_unpin_pages);
608571
609−/*
610− * This interface allows the CPUs to perform some sort of virtual DMA on
611− * behalf of the device.
612− *
613− * CPUs read/write from/into a range of IOVAs pointing to user space memory
614− * into/from a kernel buffer.
615− *
616− * As the read/write of user space memory is conducted via the CPUs and is
617− * not a real device DMA, it is not necessary to pin the user space memory.
618− *
619− * @device [in] : VFIO device
620− * @iova [in] : base IOVA of a user space buffer
621− * @data [in] : pointer to kernel buffer
622− * @len [in] : kernel buffer length
623− * @write : indicate read or write
624− * Return error code on failure or 0 on success.
625− */
626−int vfio_dma_rw(struct vfio_device *device, dma_addr_t iova, void *data,
627− size_t len, bool write)
572+int vfio_device_container_dma_rw(struct vfio_device *device,
573+ dma_addr_t iova, void *data,
574+ size_t len, bool write)
628575 {
629− struct vfio_container *container;
630− struct vfio_iommu_driver *driver;
631− int ret = 0;
632−
633− if (!data || len <= 0 || !vfio_assert_device_open(device))
634− return -EINVAL;
635−
636− /* group->container cannot change while a vfio device is open */
637− container = device->group->container;
638− driver = container->iommu_driver;
576+ struct vfio_container *container = device->group->container;
577+ struct vfio_iommu_driver *driver = container->iommu_driver;
639578
640− if (likely(driver && driver->ops->dma_rw))
641− ret = driver->ops->dma_rw(container->iommu_data,
642− iova, data, len, write);
643− else
644− ret = -ENOTTY;
645− return ret;
579+ if (unlikely(!driver || !driver->ops->dma_rw))
580+ return -ENOTTY;
581+ return driver->ops->dma_rw(container->iommu_data, iova, data, len,
582+ write);
646583 }
647−EXPORT_SYMBOL(vfio_dma_rw);
648584
649585 int __init vfio_container_init(void)
650586 {
@@ -678,3 +614,6 @@ void vfio_container_cleanup(void) · 27 unmodified lines
678614 misc_deregister(&vfio_dev);
679615 mutex_destroy(&vfio.iommu_drivers_lock);
680616 }
617+
618+MODULE_ALIAS_MISCDEV(VFIO_MINOR);
619+MODULE_ALIAS("devname:vfio/vfio");
drivers/vfio/fsl-mc/vfio_fsl_mc.c+3−0
@@ -592,6 +592,9 @@ static const struct vfio_device_ops vfio_fsl_mc_ops = { · 591 unmodified lines
592592 .read = vfio_fsl_mc_read,
593593 .write = vfio_fsl_mc_write,
594594 .mmap = vfio_fsl_mc_mmap,
595+ .bind_iommufd = vfio_iommufd_physical_bind,
596+ .unbind_iommufd = vfio_iommufd_physical_unbind,
597+ .attach_ioas = vfio_iommufd_physical_attach_ioas,
595598 };
596599
597600 static struct fsl_mc_driver vfio_fsl_mc_driver = {
drivers/vfio/group.c+877−0
@@ -0,0 +1,877 @@
1+// SPDX-License-Identifier: GPL-2.0-only
2+/*
3+ * VFIO core
4+ *
5+ * Copyright (C) 2012 Red Hat, Inc. All rights reserved.
6+ * Author: Alex Williamson <alex.williamson@redhat.com>
7+ *
8+ * Derived from original vfio:
9+ * Copyright 2010 Cisco Systems, Inc. All rights reserved.
10+ * Author: Tom Lyon, pugs@cisco.com
11+ */
12+
13+#include <linux/vfio.h>
14+#include <linux/iommufd.h>
15+#include <linux/anon_inodes.h>
16+#include "vfio.h"
17+
18+static struct vfio {
19+ struct class *class;
20+ struct list_head group_list;
21+ struct mutex group_lock; /* locks group_list */
22+ struct ida group_ida;
23+ dev_t group_devt;
24+} vfio;
25+
26+static struct vfio_device *vfio_device_get_from_name(struct vfio_group *group,
27+ char *buf)
28+{
29+ struct vfio_device *it, *device = ERR_PTR(-ENODEV);
30+
31+ mutex_lock(&group->device_lock);
32+ list_for_each_entry(it, &group->device_list, group_next) {
33+ int ret;
34+
35+ if (it->ops->match) {
36+ ret = it->ops->match(it, buf);
37+ if (ret < 0) {
38+ device = ERR_PTR(ret);
39+ break;
40+ }
41+ } else {
42+ ret = !strcmp(dev_name(it->dev), buf);
43+ }
44+
45+ if (ret && vfio_device_try_get_registration(it)) {
46+ device = it;
47+ break;
48+ }
49+ }
50+ mutex_unlock(&group->device_lock);
51+
52+ return device;
53+}
54+
55+/*
56+ * VFIO Group fd, /dev/vfio/$GROUP
57+ */
58+static bool vfio_group_has_iommu(struct vfio_group *group)
59+{
60+ lockdep_assert_held(&group->group_lock);
61+ /*
62+ * There can only be users if there is a container, and if there is a
63+ * container there must be users.
64+ */
65+ WARN_ON(!group->container != !group->container_users);
66+
67+ return group->container || group->iommufd;
68+}
69+
70+/*
71+ * VFIO_GROUP_UNSET_CONTAINER should fail if there are other users or
72+ * if there was no container to unset. Since the ioctl is called on
73+ * the group, we know that still exists, therefore the only valid
74+ * transition here is 1->0.
75+ */
76+static int vfio_group_ioctl_unset_container(struct vfio_group *group)
77+{
78+ int ret = 0;
79+
80+ mutex_lock(&group->group_lock);
81+ if (!vfio_group_has_iommu(group)) {
82+ ret = -EINVAL;
83+ goto out_unlock;
84+ }
85+ if (group->container) {
86+ if (group->container_users != 1) {
87+ ret = -EBUSY;
88+ goto out_unlock;
89+ }
90+ vfio_group_detach_container(group);
91+ }
92+ if (group->iommufd) {
93+ iommufd_ctx_put(group->iommufd);
94+ group->iommufd = NULL;
95+ }
96+
97+out_unlock:
98+ mutex_unlock(&group->group_lock);
99+ return ret;
100+}
101+
102+static int vfio_group_ioctl_set_container(struct vfio_group *group,
103+ int __user *arg)
104+{
105+ struct vfio_container *container;
106+ struct iommufd_ctx *iommufd;
107+ struct fd f;
108+ int ret;
109+ int fd;
110+
111+ if (get_user(fd, arg))
112+ return -EFAULT;
113+
114+ f = fdget(fd);
115+ if (!f.file)
116+ return -EBADF;
117+
118+ mutex_lock(&group->group_lock);
119+ if (vfio_group_has_iommu(group)) {
120+ ret = -EINVAL;
121+ goto out_unlock;
122+ }
123+ if (!group->iommu_group) {
124+ ret = -ENODEV;
125+ goto out_unlock;
126+ }
127+
128+ container = vfio_container_from_file(f.file);
129+ if (container) {
130+ ret = vfio_container_attach_group(container, group);
131+ goto out_unlock;
132+ }
133+
134+ iommufd = iommufd_ctx_from_file(f.file);
135+ if (!IS_ERR(iommufd)) {
136+ u32 ioas_id;
137+
138+ ret = iommufd_vfio_compat_ioas_id(iommufd, &ioas_id);
139+ if (ret) {
140+ iommufd_ctx_put(group->iommufd);
141+ goto out_unlock;
142+ }
143+
144+ group->iommufd = iommufd;
145+ goto out_unlock;
146+ }
147+
148+ /* The FD passed is not recognized. */
149+ ret = -EBADFD;
150+
151+out_unlock:
152+ mutex_unlock(&group->group_lock);
153+ fdput(f);
154+ return ret;
155+}
156+
157+static int vfio_device_group_open(struct vfio_device *device)
158+{
159+ int ret;
160+
161+ mutex_lock(&device->group->group_lock);
162+ if (!vfio_group_has_iommu(device->group)) {
163+ ret = -EINVAL;
164+ goto out_unlock;
165+ }
166+
167+ /*
168+ * Here we pass the KVM pointer with the group under the lock. If the
169+ * device driver will use it, it must obtain a reference and release it
170+ * during close_device.
171+ */
172+ ret = vfio_device_open(device, device->group->iommufd,
173+ device->group->kvm);
174+
175+out_unlock:
176+ mutex_unlock(&device->group->group_lock);
177+ return ret;
178+}
179+
180+void vfio_device_group_close(struct vfio_device *device)
181+{
182+ mutex_lock(&device->group->group_lock);
183+ vfio_device_close(device, device->group->iommufd);
184+ mutex_unlock(&device->group->group_lock);
185+}
186+
187+static struct file *vfio_device_open_file(struct vfio_device *device)
188+{
189+ struct file *filep;
190+ int ret;
191+
192+ ret = vfio_device_group_open(device);
193+ if (ret)
194+ goto err_out;
195+
196+ /*
197+ * We can't use anon_inode_getfd() because we need to modify
198+ * the f_mode flags directly to allow more than just ioctls
199+ */
200+ filep = anon_inode_getfile("[vfio-device]", &vfio_device_fops,
201+ device, O_RDWR);
202+ if (IS_ERR(filep)) {
203+ ret = PTR_ERR(filep);
204+ goto err_close_device;
205+ }
206+
207+ /*
208+ * TODO: add an anon_inode interface to do this.
209+ * Appears to be missing by lack of need rather than
210+ * explicitly prevented. Now there's need.
211+ */
212+ filep->f_mode |= (FMODE_PREAD | FMODE_PWRITE);
213+
214+ if (device->group->type == VFIO_NO_IOMMU)
215+ dev_warn(device->dev, "vfio-noiommu device opened by user "
216+ "(%s:%d)\n", current->comm, task_pid_nr(current));
217+ /*
218+ * On success the ref of device is moved to the file and
219+ * put in vfio_device_fops_release()
220+ */
221+ return filep;
222+
223+err_close_device:
224+ vfio_device_group_close(device);
225+err_out:
226+ return ERR_PTR(ret);
227+}
228+
229+static int vfio_group_ioctl_get_device_fd(struct vfio_group *group,
230+ char __user *arg)
231+{
232+ struct vfio_device *device;
233+ struct file *filep;
234+ char *buf;
235+ int fdno;
236+ int ret;
237+
238+ buf = strndup_user(arg, PAGE_SIZE);
239+ if (IS_ERR(buf))
240+ return PTR_ERR(buf);
241+
242+ device = vfio_device_get_from_name(group, buf);
243+ kfree(buf);
244+ if (IS_ERR(device))
245+ return PTR_ERR(device);
246+
247+ fdno = get_unused_fd_flags(O_CLOEXEC);
248+ if (fdno < 0) {
249+ ret = fdno;
250+ goto err_put_device;
251+ }
252+
253+ filep = vfio_device_open_file(device);
254+ if (IS_ERR(filep)) {
255+ ret = PTR_ERR(filep);
256+ goto err_put_fdno;
257+ }
258+
259+ fd_install(fdno, filep);
260+ return fdno;
261+
262+err_put_fdno:
263+ put_unused_fd(fdno);
264+err_put_device:
265+ vfio_device_put_registration(device);
266+ return ret;
267+}
268+
269+static int vfio_group_ioctl_get_status(struct vfio_group *group,
270+ struct vfio_group_status __user *arg)
271+{
272+ unsigned long minsz = offsetofend(struct vfio_group_status, flags);
273+ struct vfio_group_status status;
274+
275+ if (copy_from_user(&status, arg, minsz))
276+ return -EFAULT;
277+
278+ if (status.argsz < minsz)
279+ return -EINVAL;
280+
281+ status.flags = 0;
282+
283+ mutex_lock(&group->group_lock);
284+ if (!group->iommu_group) {
285+ mutex_unlock(&group->group_lock);
286+ return -ENODEV;
287+ }
288+
289+ /*
290+ * With the container FD the iommu_group_claim_dma_owner() is done
291+ * during SET_CONTAINER but for IOMMFD this is done during
292+ * VFIO_GROUP_GET_DEVICE_FD. Meaning that with iommufd
293+ * VFIO_GROUP_FLAGS_VIABLE could be set but GET_DEVICE_FD will fail due
294+ * to viability.
295+ */
296+ if (vfio_group_has_iommu(group))
297+ status.flags |= VFIO_GROUP_FLAGS_CONTAINER_SET |
298+ VFIO_GROUP_FLAGS_VIABLE;
299+ else if (!iommu_group_dma_owner_claimed(group->iommu_group))
300+ status.flags |= VFIO_GROUP_FLAGS_VIABLE;
301+ mutex_unlock(&group->group_lock);
302+
303+ if (copy_to_user(arg, &status, minsz))
304+ return -EFAULT;
305+ return 0;
306+}
307+
308+static long vfio_group_fops_unl_ioctl(struct file *filep,
309+ unsigned int cmd, unsigned long arg)
310+{
311+ struct vfio_group *group = filep->private_data;
312+ void __user *uarg = (void __user *)arg;
313+
314+ switch (cmd) {
315+ case VFIO_GROUP_GET_DEVICE_FD:
316+ return vfio_group_ioctl_get_device_fd(group, uarg);
317+ case VFIO_GROUP_GET_STATUS:
318+ return vfio_group_ioctl_get_status(group, uarg);
319+ case VFIO_GROUP_SET_CONTAINER:
320+ return vfio_group_ioctl_set_container(group, uarg);
321+ case VFIO_GROUP_UNSET_CONTAINER:
322+ return vfio_group_ioctl_unset_container(group);
323+ default:
324+ return -ENOTTY;
325+ }
326+}
327+
328+static int vfio_group_fops_open(struct inode *inode, struct file *filep)
329+{
330+ struct vfio_group *group =
331+ container_of(inode->i_cdev, struct vfio_group, cdev);
332+ int ret;
333+
334+ mutex_lock(&group->group_lock);
335+
336+ /*
337+ * drivers can be zero if this races with vfio_device_remove_group(), it
338+ * will be stable at 0 under the group rwsem
339+ */
340+ if (refcount_read(&group->drivers) == 0) {
341+ ret = -ENODEV;
342+ goto out_unlock;
343+ }
344+
345+ if (group->type == VFIO_NO_IOMMU && !capable(CAP_SYS_RAWIO)) {
346+ ret = -EPERM;
347+ goto out_unlock;
348+ }
349+
350+ /*
351+ * Do we need multiple instances of the group open? Seems not.
352+ */
353+ if (group->opened_file) {
354+ ret = -EBUSY;
355+ goto out_unlock;
356+ }
357+ group->opened_file = filep;
358+ filep->private_data = group;
359+ ret = 0;
360+out_unlock:
361+ mutex_unlock(&group->group_lock);
362+ return ret;
363+}
364+
365+static int vfio_group_fops_release(struct inode *inode, struct file *filep)
366+{
367+ struct vfio_group *group = filep->private_data;
368+
369+ filep->private_data = NULL;
370+
371+ mutex_lock(&group->group_lock);
372+ /*
373+ * Device FDs hold a group file reference, therefore the group release
374+ * is only called when there are no open devices.
375+ */
376+ WARN_ON(group->notifier.head);
377+ if (group->container)
378+ vfio_group_detach_container(group);
379+ if (group->iommufd) {
380+ iommufd_ctx_put(group->iommufd);
381+ group->iommufd = NULL;
382+ }
383+ group->opened_file = NULL;
384+ mutex_unlock(&group->group_lock);
385+ return 0;
386+}
387+
388+static const struct file_operations vfio_group_fops = {
389+ .owner = THIS_MODULE,
390+ .unlocked_ioctl = vfio_group_fops_unl_ioctl,
391+ .compat_ioctl = compat_ptr_ioctl,
392+ .open = vfio_group_fops_open,
393+ .release = vfio_group_fops_release,
394+};
395+
396+/*
397+ * Group objects - create, release, get, put, search
398+ */
399+static struct vfio_group *
400+vfio_group_find_from_iommu(struct iommu_group *iommu_group)
401+{
402+ struct vfio_group *group;
403+
404+ lockdep_assert_held(&vfio.group_lock);
405+
406+ /*
407+ * group->iommu_group from the vfio.group_list cannot be NULL
408+ * under the vfio.group_lock.
409+ */
410+ list_for_each_entry(group, &vfio.group_list, vfio_next) {
411+ if (group->iommu_group == iommu_group)
412+ return group;
413+ }
414+ return NULL;
415+}
416+
417+static void vfio_group_release(struct device *dev)
418+{
419+ struct vfio_group *group = container_of(dev, struct vfio_group, dev);
420+
421+ mutex_destroy(&group->device_lock);
422+ mutex_destroy(&group->group_lock);
423+ WARN_ON(group->iommu_group);
424+ ida_free(&vfio.group_ida, MINOR(group->dev.devt));
425+ kfree(group);
426+}
427+
428+static struct vfio_group *vfio_group_alloc(struct iommu_group *iommu_group,
429+ enum vfio_group_type type)
430+{
431+ struct vfio_group *group;
432+ int minor;
433+
434+ group = kzalloc(sizeof(*group), GFP_KERNEL);
435+ if (!group)
436+ return ERR_PTR(-ENOMEM);
437+
438+ minor = ida_alloc_max(&vfio.group_ida, MINORMASK, GFP_KERNEL);
439+ if (minor < 0) {
440+ kfree(group);
441+ return ERR_PTR(minor);
442+ }
443+
444+ device_initialize(&group->dev);
445+ group->dev.devt = MKDEV(MAJOR(vfio.group_devt), minor);
446+ group->dev.class = vfio.class;
447+ group->dev.release = vfio_group_release;
448+ cdev_init(&group->cdev, &vfio_group_fops);
449+ group->cdev.owner = THIS_MODULE;
450+
451+ refcount_set(&group->drivers, 1);
452+ mutex_init(&group->group_lock);
453+ INIT_LIST_HEAD(&group->device_list);
454+ mutex_init(&group->device_lock);
455+ group->iommu_group = iommu_group;
456+ /* put in vfio_group_release() */
457+ iommu_group_ref_get(iommu_group);
458+ group->type = type;
459+ BLOCKING_INIT_NOTIFIER_HEAD(&group->notifier);
460+
461+ return group;
462+}
463+
464+static struct vfio_group *vfio_create_group(struct iommu_group *iommu_group,
465+ enum vfio_group_type type)
466+{
467+ struct vfio_group *group;
468+ struct vfio_group *ret;
469+ int err;
470+
471+ lockdep_assert_held(&vfio.group_lock);
472+
473+ group = vfio_group_alloc(iommu_group, type);
474+ if (IS_ERR(group))
475+ return group;
476+
477+ err = dev_set_name(&group->dev, "%s%d",
478+ group->type == VFIO_NO_IOMMU ? "noiommu-" : "",
479+ iommu_group_id(iommu_group));
480+ if (err) {
481+ ret = ERR_PTR(err);
482+ goto err_put;
483+ }
484+
485+ err = cdev_device_add(&group->cdev, &group->dev);
486+ if (err) {
487+ ret = ERR_PTR(err);
488+ goto err_put;
489+ }
490+
491+ list_add(&group->vfio_next, &vfio.group_list);
492+
493+ return group;
494+
495+err_put:
496+ put_device(&group->dev);
497+ return ret;
498+}
499+
500+static struct vfio_group *vfio_noiommu_group_alloc(struct device *dev,
501+ enum vfio_group_type type)
502+{
503+ struct iommu_group *iommu_group;
504+ struct vfio_group *group;
505+ int ret;
506+
507+ iommu_group = iommu_group_alloc();
508+ if (IS_ERR(iommu_group))
509+ return ERR_CAST(iommu_group);
510+
511+ ret = iommu_group_set_name(iommu_group, "vfio-noiommu");
512+ if (ret)
513+ goto out_put_group;
514+ ret = iommu_group_add_device(iommu_group, dev);
515+ if (ret)
516+ goto out_put_group;
517+
518+ mutex_lock(&vfio.group_lock);
519+ group = vfio_create_group(iommu_group, type);
520+ mutex_unlock(&vfio.group_lock);
521+ if (IS_ERR(group)) {
522+ ret = PTR_ERR(group);
523+ goto out_remove_device;
524+ }
525+ iommu_group_put(iommu_group);
526+ return group;
527+
528+out_remove_device:
529+ iommu_group_remove_device(dev);
530+out_put_group:
531+ iommu_group_put(iommu_group);
532+ return ERR_PTR(ret);
533+}
534+
535+static bool vfio_group_has_device(struct vfio_group *group, struct device *dev)
536+{
537+ struct vfio_device *device;
538+
539+ mutex_lock(&group->device_lock);
540+ list_for_each_entry(device, &group->device_list, group_next) {
541+ if (device->dev == dev) {
542+ mutex_unlock(&group->device_lock);
543+ return true;
544+ }
545+ }
546+ mutex_unlock(&group->device_lock);
547+ return false;
548+}
549+
550+static struct vfio_group *vfio_group_find_or_alloc(struct device *dev)
551+{
552+ struct iommu_group *iommu_group;
553+ struct vfio_group *group;
554+
555+ iommu_group = iommu_group_get(dev);
556+ if (!iommu_group && vfio_noiommu) {
557+ /*
558+ * With noiommu enabled, create an IOMMU group for devices that
559+ * don't already have one, implying no IOMMU hardware/driver
560+ * exists. Taint the kernel because we're about to give a DMA
561+ * capable device to a user without IOMMU protection.
562+ */
563+ group = vfio_noiommu_group_alloc(dev, VFIO_NO_IOMMU);
564+ if (!IS_ERR(group)) {
565+ add_taint(TAINT_USER, LOCKDEP_STILL_OK);
566+ dev_warn(dev, "Adding kernel taint for vfio-noiommu group on device\n");
567+ }
568+ return group;
569+ }
570+
571+ if (!iommu_group)
572+ return ERR_PTR(-EINVAL);
573+
574+ /*
575+ * VFIO always sets IOMMU_CACHE because we offer no way for userspace to
576+ * restore cache coherency. It has to be checked here because it is only
577+ * valid for cases where we are using iommu groups.
578+ */
579+ if (!device_iommu_capable(dev, IOMMU_CAP_CACHE_COHERENCY)) {
580+ iommu_group_put(iommu_group);
581+ return ERR_PTR(-EINVAL);
582+ }
583+
584+ mutex_lock(&vfio.group_lock);
585+ group = vfio_group_find_from_iommu(iommu_group);
586+ if (group) {
587+ if (WARN_ON(vfio_group_has_device(group, dev)))
588+ group = ERR_PTR(-EINVAL);
589+ else
590+ refcount_inc(&group->drivers);
591+ } else {
592+ group = vfio_create_group(iommu_group, VFIO_IOMMU);
593+ }
594+ mutex_unlock(&vfio.group_lock);
595+
596+ /* The vfio_group holds a reference to the iommu_group */
597+ iommu_group_put(iommu_group);
598+ return group;
599+}
600+
601+int vfio_device_set_group(struct vfio_device *device,
602+ enum vfio_group_type type)
603+{
604+ struct vfio_group *group;
605+
606+ if (type == VFIO_IOMMU)
607+ group = vfio_group_find_or_alloc(device->dev);
608+ else
609+ group = vfio_noiommu_group_alloc(device->dev, type);
610+
611+ if (IS_ERR(group))
612+ return PTR_ERR(group);
613+
614+ /* Our reference on group is moved to the device */
615+ device->group = group;
616+ return 0;
617+}
618+
619+void vfio_device_remove_group(struct vfio_device *device)
620+{
621+ struct vfio_group *group = device->group;
622+ struct iommu_group *iommu_group;
623+
624+ if (group->type == VFIO_NO_IOMMU || group->type == VFIO_EMULATED_IOMMU)
625+ iommu_group_remove_device(device->dev);
626+
627+ /* Pairs with vfio_create_group() / vfio_group_get_from_iommu() */
628+ if (!refcount_dec_and_mutex_lock(&group->drivers, &vfio.group_lock))
629+ return;
630+ list_del(&group->vfio_next);
631+
632+ /*
633+ * We could concurrently probe another driver in the group that might
634+ * race vfio_device_remove_group() with vfio_get_group(), so we have to
635+ * ensure that the sysfs is all cleaned up under lock otherwise the
636+ * cdev_device_add() will fail due to the name aready existing.
637+ */
638+ cdev_device_del(&group->cdev, &group->dev);
639+
640+ mutex_lock(&group->group_lock);
641+ /*
642+ * These data structures all have paired operations that can only be
643+ * undone when the caller holds a live reference on the device. Since
644+ * all pairs must be undone these WARN_ON's indicate some caller did not
645+ * properly hold the group reference.
646+ */
647+ WARN_ON(!list_empty(&group->device_list));
648+ WARN_ON(group->notifier.head);
649+
650+ /*
651+ * Revoke all users of group->iommu_group. At this point we know there
652+ * are no devices active because we are unplugging the last one. Setting
653+ * iommu_group to NULL blocks all new users.
654+ */
655+ if (group->container)
656+ vfio_group_detach_container(group);
657+ iommu_group = group->iommu_group;
658+ group->iommu_group = NULL;
659+ mutex_unlock(&group->group_lock);
660+ mutex_unlock(&vfio.group_lock);
661+
662+ iommu_group_put(iommu_group);
663+ put_device(&group->dev);
664+}
665+
666+void vfio_device_group_register(struct vfio_device *device)
667+{
668+ mutex_lock(&device->group->device_lock);
669+ list_add(&device->group_next, &device->group->device_list);
670+ mutex_unlock(&device->group->device_lock);
671+}
672+
673+void vfio_device_group_unregister(struct vfio_device *device)
674+{
675+ mutex_lock(&device->group->device_lock);
676+ list_del(&device->group_next);
677+ mutex_unlock(&device->group->device_lock);
678+}
679+
680+int vfio_device_group_use_iommu(struct vfio_device *device)
681+{
682+ struct vfio_group *group = device->group;
683+ int ret = 0;
684+
685+ lockdep_assert_held(&group->group_lock);
686+
687+ if (WARN_ON(!group->container))
688+ return -EINVAL;
689+
690+ ret = vfio_group_use_container(group);
691+ if (ret)
692+ return ret;
693+ vfio_device_container_register(device);
694+ return 0;
695+}
696+
697+void vfio_device_group_unuse_iommu(struct vfio_device *device)
698+{
699+ struct vfio_group *group = device->group;
700+
701+ lockdep_assert_held(&group->group_lock);
702+
703+ if (WARN_ON(!group->container))
704+ return;
705+
706+ vfio_device_container_unregister(device);
707+ vfio_group_unuse_container(group);
708+}
709+
710+bool vfio_device_has_container(struct vfio_device *device)
711+{
712+ return device->group->container;
713+}
714+
715+/**
716+ * vfio_file_iommu_group - Return the struct iommu_group for the vfio group file
717+ * @file: VFIO group file
718+ *
719+ * The returned iommu_group is valid as long as a ref is held on the file. This
720+ * returns a reference on the group. This function is deprecated, only the SPAPR
721+ * path in kvm should call it.
722+ */
723+struct iommu_group *vfio_file_iommu_group(struct file *file)
724+{
725+ struct vfio_group *group = file->private_data;
726+ struct iommu_group *iommu_group = NULL;
727+
728+ if (!IS_ENABLED(CONFIG_SPAPR_TCE_IOMMU))
729+ return NULL;
730+
731+ if (!vfio_file_is_group(file))
732+ return NULL;
733+
734+ mutex_lock(&group->group_lock);
735+ if (group->iommu_group) {
736+ iommu_group = group->iommu_group;
737+ iommu_group_ref_get(iommu_group);
738+ }
739+ mutex_unlock(&group->group_lock);
740+ return iommu_group;
741+}
742+EXPORT_SYMBOL_GPL(vfio_file_iommu_group);
743+
744+/**
745+ * vfio_file_is_group - True if the file is usable with VFIO aPIS
746+ * @file: VFIO group file
747+ */
748+bool vfio_file_is_group(struct file *file)
749+{
750+ return file->f_op == &vfio_group_fops;
751+}
752+EXPORT_SYMBOL_GPL(vfio_file_is_group);
753+
754+/**
755+ * vfio_file_enforced_coherent - True if the DMA associated with the VFIO file
756+ * is always CPU cache coherent
757+ * @file: VFIO group file
758+ *
759+ * Enforced coherency means that the IOMMU ignores things like the PCIe no-snoop
760+ * bit in DMA transactions. A return of false indicates that the user has
761+ * rights to access additional instructions such as wbinvd on x86.
762+ */
763+bool vfio_file_enforced_coherent(struct file *file)
764+{
765+ struct vfio_group *group = file->private_data;
766+ struct vfio_device *device;
767+ bool ret = true;
768+
769+ if (!vfio_file_is_group(file))
770+ return true;
771+
772+ /*
773+ * If the device does not have IOMMU_CAP_ENFORCE_CACHE_COHERENCY then
774+ * any domain later attached to it will also not support it. If the cap
775+ * is set then the iommu_domain eventually attached to the device/group
776+ * must use a domain with enforce_cache_coherency().
777+ */
778+ mutex_lock(&group->device_lock);
779+ list_for_each_entry(device, &group->device_list, group_next) {
780+ if (!device_iommu_capable(device->dev,
781+ IOMMU_CAP_ENFORCE_CACHE_COHERENCY)) {
782+ ret = false;
783+ break;
784+ }
785+ }
786+ mutex_unlock(&group->device_lock);
787+ return ret;
788+}
789+EXPORT_SYMBOL_GPL(vfio_file_enforced_coherent);
790+
791+/**
792+ * vfio_file_set_kvm - Link a kvm with VFIO drivers
793+ * @file: VFIO group file
794+ * @kvm: KVM to link
795+ *
796+ * When a VFIO device is first opened the KVM will be available in
797+ * device->kvm if one was associated with the group.
798+ */
799+void vfio_file_set_kvm(struct file *file, struct kvm *kvm)
800+{
801+ struct vfio_group *group = file->private_data;
802+
803+ if (!vfio_file_is_group(file))
804+ return;
805+
806+ mutex_lock(&group->group_lock);
807+ group->kvm = kvm;
808+ mutex_unlock(&group->group_lock);
809+}
810+EXPORT_SYMBOL_GPL(vfio_file_set_kvm);
811+
812+/**
813+ * vfio_file_has_dev - True if the VFIO file is a handle for device
814+ * @file: VFIO file to check
815+ * @device: Device that must be part of the file
816+ *
817+ * Returns true if given file has permission to manipulate the given device.
818+ */
819+bool vfio_file_has_dev(struct file *file, struct vfio_device *device)
820+{
821+ struct vfio_group *group = file->private_data;
822+
823+ if (!vfio_file_is_group(file))
824+ return false;
825+
826+ return group == device->group;
827+}
828+EXPORT_SYMBOL_GPL(vfio_file_has_dev);
829+
830+static char *vfio_devnode(struct device *dev, umode_t *mode)
831+{
832+ return kasprintf(GFP_KERNEL, "vfio/%s", dev_name(dev));
833+}
834+
835+int __init vfio_group_init(void)
836+{
837+ int ret;
838+
839+ ida_init(&vfio.group_ida);
840+ mutex_init(&vfio.group_lock);
841+ INIT_LIST_HEAD(&vfio.group_list);
842+
843+ ret = vfio_container_init();
844+ if (ret)
845+ return ret;
846+
847+ /* /dev/vfio/$GROUP */
848+ vfio.class = class_create(THIS_MODULE, "vfio");
849+ if (IS_ERR(vfio.class)) {
850+ ret = PTR_ERR(vfio.class);
851+ goto err_group_class;
852+ }
853+
854+ vfio.class->devnode = vfio_devnode;
855+
856+ ret = alloc_chrdev_region(&vfio.group_devt, 0, MINORMASK + 1, "vfio");
857+ if (ret)
858+ goto err_alloc_chrdev;
859+ return 0;
860+
861+err_alloc_chrdev:
862+ class_destroy(vfio.class);
863+ vfio.class = NULL;
864+err_group_class:
865+ vfio_container_cleanup();
866+ return ret;
867+}
868+
869+void vfio_group_cleanup(void)
870+{
871+ WARN_ON(!list_empty(&vfio.group_list));
872+ ida_destroy(&vfio.group_ida);
873+ unregister_chrdev_region(vfio.group_devt, MINORMASK + 1);
874+ class_destroy(vfio.class);
875+ vfio.class = NULL;
876+ vfio_container_cleanup();
877+}
drivers/vfio/iommufd.c+158−0
@@ -0,0 +1,158 @@
1+// SPDX-License-Identifier: GPL-2.0-only
2+/*
3+ * Copyright (c) 2021-2022, NVIDIA CORPORATION & AFFILIATES
4+ */
5+#include <linux/vfio.h>
6+#include <linux/iommufd.h>
7+
8+#include "vfio.h"
9+
10+MODULE_IMPORT_NS(IOMMUFD);
11+MODULE_IMPORT_NS(IOMMUFD_VFIO);
12+
13+int vfio_iommufd_bind(struct vfio_device *vdev, struct iommufd_ctx *ictx)
14+{
15+ u32 ioas_id;
16+ u32 device_id;
17+ int ret;
18+
19+ lockdep_assert_held(&vdev->dev_set->lock);
20+
21+ /*
22+ * If the driver doesn't provide this op then it means the device does
23+ * not do DMA at all. So nothing to do.
24+ */
25+ if (!vdev->ops->bind_iommufd)
26+ return 0;
27+
28+ ret = vdev->ops->bind_iommufd(vdev, ictx, &device_id);
29+ if (ret)
30+ return ret;
31+
32+ ret = iommufd_vfio_compat_ioas_id(ictx, &ioas_id);
33+ if (ret)
34+ goto err_unbind;
35+ ret = vdev->ops->attach_ioas(vdev, &ioas_id);
36+ if (ret)
37+ goto err_unbind;
38+
39+ /*
40+ * The legacy path has no way to return the device id or the selected
41+ * pt_id
42+ */
43+ return 0;
44+
45+err_unbind:
46+ if (vdev->ops->unbind_iommufd)
47+ vdev->ops->unbind_iommufd(vdev);
48+ return ret;
49+}
50+
51+void vfio_iommufd_unbind(struct vfio_device *vdev)
52+{
53+ lockdep_assert_held(&vdev->dev_set->lock);
54+
55+ if (vdev->ops->unbind_iommufd)
56+ vdev->ops->unbind_iommufd(vdev);
57+}
58+
59+/*
60+ * The physical standard ops mean that the iommufd_device is bound to the
61+ * physical device vdev->dev that was provided to vfio_init_group_dev(). Drivers
62+ * using this ops set should call vfio_register_group_dev()
63+ */
64+int vfio_iommufd_physical_bind(struct vfio_device *vdev,
65+ struct iommufd_ctx *ictx, u32 *out_device_id)
66+{
67+ struct iommufd_device *idev;
68+
69+ idev = iommufd_device_bind(ictx, vdev->dev, out_device_id);
70+ if (IS_ERR(idev))
71+ return PTR_ERR(idev);
72+ vdev->iommufd_device = idev;
73+ return 0;
74+}
75+EXPORT_SYMBOL_GPL(vfio_iommufd_physical_bind);
76+
77+void vfio_iommufd_physical_unbind(struct vfio_device *vdev)
78+{
79+ lockdep_assert_held(&vdev->dev_set->lock);
80+
81+ if (vdev->iommufd_attached) {
82+ iommufd_device_detach(vdev->iommufd_device);
83+ vdev->iommufd_attached = false;
84+ }
85+ iommufd_device_unbind(vdev->iommufd_device);
86+ vdev->iommufd_device = NULL;
87+}
88+EXPORT_SYMBOL_GPL(vfio_iommufd_physical_unbind);
89+
90+int vfio_iommufd_physical_attach_ioas(struct vfio_device *vdev, u32 *pt_id)
91+{
92+ int rc;
93+
94+ rc = iommufd_device_attach(vdev->iommufd_device, pt_id);
95+ if (rc)
96+ return rc;
97+ vdev->iommufd_attached = true;
98+ return 0;
99+}
100+EXPORT_SYMBOL_GPL(vfio_iommufd_physical_attach_ioas);
101+
102+/*
103+ * The emulated standard ops mean that vfio_device is going to use the
104+ * "mdev path" and will call vfio_pin_pages()/vfio_dma_rw(). Drivers using this
105+ * ops set should call vfio_register_emulated_iommu_dev().
106+ */
107+
108+static void vfio_emulated_unmap(void *data, unsigned long iova,
109+ unsigned long length)
110+{
111+ struct vfio_device *vdev = data;
112+
113+ vdev->ops->dma_unmap(vdev, iova, length);
114+}
115+
116+static const struct iommufd_access_ops vfio_user_ops = {
117+ .needs_pin_pages = 1,
118+ .unmap = vfio_emulated_unmap,
119+};
120+
121+int vfio_iommufd_emulated_bind(struct vfio_device *vdev,
122+ struct iommufd_ctx *ictx, u32 *out_device_id)
123+{
124+ lockdep_assert_held(&vdev->dev_set->lock);
125+
126+ vdev->iommufd_ictx = ictx;
127+ iommufd_ctx_get(ictx);
128+ return 0;
129+}
130+EXPORT_SYMBOL_GPL(vfio_iommufd_emulated_bind);
131+
132+void vfio_iommufd_emulated_unbind(struct vfio_device *vdev)
133+{
134+ lockdep_assert_held(&vdev->dev_set->lock);
135+
136+ if (vdev->iommufd_access) {
137+ iommufd_access_destroy(vdev->iommufd_access);
138+ vdev->iommufd_access = NULL;
139+ }
140+ iommufd_ctx_put(vdev->iommufd_ictx);
141+ vdev->iommufd_ictx = NULL;
142+}
143+EXPORT_SYMBOL_GPL(vfio_iommufd_emulated_unbind);
144+
145+int vfio_iommufd_emulated_attach_ioas(struct vfio_device *vdev, u32 *pt_id)
146+{
147+ struct iommufd_access *user;
148+
149+ lockdep_assert_held(&vdev->dev_set->lock);
150+
151+ user = iommufd_access_create(vdev->iommufd_ictx, *pt_id, &vfio_user_ops,
152+ vdev);
153+ if (IS_ERR(user))
154+ return PTR_ERR(user);
155+ vdev->iommufd_access = user;
156+ return 0;
157+}
158+EXPORT_SYMBOL_GPL(vfio_iommufd_emulated_attach_ioas);
drivers/vfio/pci/hisilicon/hisi_acc_vfio_pci.c+6−0
@@ -1246,6 +1246,9 @@ static const struct vfio_device_ops hisi_acc_vfio_pci_migrn_ops = { · 1245 unmodified lines
12461246 .mmap = hisi_acc_vfio_pci_mmap,
12471247 .request = vfio_pci_core_request,
12481248 .match = vfio_pci_core_match,
1249+ .bind_iommufd = vfio_iommufd_physical_bind,
1250+ .unbind_iommufd = vfio_iommufd_physical_unbind,
1251+ .attach_ioas = vfio_iommufd_physical_attach_ioas,
12491252 };
12501253
12511254 static const struct vfio_device_ops hisi_acc_vfio_pci_ops = {
@@ -1261,6 +1264,9 @@ static const struct vfio_device_ops hisi_acc_vfio_pci_ops = { · 9 unmodified lines
12611264 .mmap = vfio_pci_core_mmap,
12621265 .request = vfio_pci_core_request,
12631266 .match = vfio_pci_core_match,
1267+ .bind_iommufd = vfio_iommufd_physical_bind,
1268+ .unbind_iommufd = vfio_iommufd_physical_unbind,
1269+ .attach_ioas = vfio_iommufd_physical_attach_ioas,
12641270 };
12651271
12661272 static int hisi_acc_vfio_pci_probe(struct pci_dev *pdev, const struct pci_device_id *id)
drivers/vfio/pci/mlx5/main.c+3−0
@@ -623,6 +623,9 @@ static const struct vfio_device_ops mlx5vf_pci_ops = { · 622 unmodified lines
623623 .mmap = vfio_pci_core_mmap,
624624 .request = vfio_pci_core_request,
625625 .match = vfio_pci_core_match,
626+ .bind_iommufd = vfio_iommufd_physical_bind,
627+ .unbind_iommufd = vfio_iommufd_physical_unbind,
628+ .attach_ioas = vfio_iommufd_physical_attach_ioas,
626629 };
627630
628631 static int mlx5vf_pci_probe(struct pci_dev *pdev,
drivers/vfio/pci/vfio_pci.c+3−0
@@ -138,6 +138,9 @@ static const struct vfio_device_ops vfio_pci_ops = { · 137 unmodified lines
138138 .mmap = vfio_pci_core_mmap,
139139 .request = vfio_pci_core_request,
140140 .match = vfio_pci_core_match,
141+ .bind_iommufd = vfio_iommufd_physical_bind,
142+ .unbind_iommufd = vfio_iommufd_physical_unbind,
143+ .attach_ioas = vfio_iommufd_physical_attach_ioas,
141144 };
142145
143146 static int vfio_pci_probe(struct pci_dev *pdev, const struct pci_device_id *id)
drivers/vfio/platform/vfio_amba.c+3−0
@@ -117,6 +117,9 @@ static const struct vfio_device_ops vfio_amba_ops = { · 116 unmodified lines
117117 .read = vfio_platform_read,
118118 .write = vfio_platform_write,
119119 .mmap = vfio_platform_mmap,
120+ .bind_iommufd = vfio_iommufd_physical_bind,
121+ .unbind_iommufd = vfio_iommufd_physical_unbind,
122+ .attach_ioas = vfio_iommufd_physical_attach_ioas,
120123 };
121124
122125 static const struct amba_id pl330_ids[] = {
drivers/vfio/platform/vfio_platform.c+3−0
@@ -106,6 +106,9 @@ static const struct vfio_device_ops vfio_platform_ops = { · 105 unmodified lines
106106 .read = vfio_platform_read,
107107 .write = vfio_platform_write,
108108 .mmap = vfio_platform_mmap,
109+ .bind_iommufd = vfio_iommufd_physical_bind,
110+ .unbind_iommufd = vfio_iommufd_physical_unbind,
111+ .attach_ioas = vfio_iommufd_physical_attach_ioas,
109112 };
110113
111114 static struct platform_driver vfio_platform_driver = {
drivers/vfio/vfio.h+114−6
@@ -6,14 +6,25 @@ · 5 unmodified lines
66 #ifndef __VFIO_VFIO_H__
77 #define __VFIO_VFIO_H__
88
9+#include <linux/file.h>
910 #include <linux/device.h>
1011 #include <linux/cdev.h>
1112 #include <linux/module.h>
1213
14+struct iommufd_ctx;
1315 struct iommu_group;
1416 struct vfio_device;
1517 struct vfio_container;
1618
19+void vfio_device_put_registration(struct vfio_device *device);
20+bool vfio_device_try_get_registration(struct vfio_device *device);
21+int vfio_device_open(struct vfio_device *device,
22+ struct iommufd_ctx *iommufd, struct kvm *kvm);
23+void vfio_device_close(struct vfio_device *device,
24+ struct iommufd_ctx *iommufd);
25+
26+extern const struct file_operations vfio_device_fops;
27+
1728 enum vfio_group_type {
1829 /*
1930 * Physical device with IOMMU backing.
@@ -54,14 +65,30 @@ struct vfio_group { · 34 unmodified lines
5465 struct list_head device_list;
5566 struct mutex device_lock;
5667 struct list_head vfio_next;
68+#if IS_ENABLED(CONFIG_VFIO_CONTAINER)
5769 struct list_head container_next;
70+#endif
5871 enum vfio_group_type type;
5972 struct mutex group_lock;
6073 struct kvm *kvm;
6174 struct file *opened_file;
6275 struct blocking_notifier_head notifier;
76+ struct iommufd_ctx *iommufd;
6377 };
6478
79+int vfio_device_set_group(struct vfio_device *device,
80+ enum vfio_group_type type);
81+void vfio_device_remove_group(struct vfio_device *device);
82+void vfio_device_group_register(struct vfio_device *device);
83+void vfio_device_group_unregister(struct vfio_device *device);
84+int vfio_device_group_use_iommu(struct vfio_device *device);
85+void vfio_device_group_unuse_iommu(struct vfio_device *device);
86+void vfio_device_group_close(struct vfio_device *device);
87+bool vfio_device_has_container(struct vfio_device *device);
88+int __init vfio_group_init(void);
89+void vfio_group_cleanup(void);
90+
91+#if IS_ENABLED(CONFIG_VFIO_CONTAINER)
6592 /* events for the backend driver notify callback */
6693 enum vfio_iommu_notify_type {
6794 VFIO_IOMMU_CONTAINER_CLOSE = 0,
@@ -109,20 +136,101 @@ struct vfio_iommu_driver { · 41 unmodified lines
109136 int vfio_register_iommu_driver(const struct vfio_iommu_driver_ops *ops);
110137 void vfio_unregister_iommu_driver(const struct vfio_iommu_driver_ops *ops);
111138
112−bool vfio_assert_device_open(struct vfio_device *device);
113−
114139 struct vfio_container *vfio_container_from_file(struct file *filep);
115−int vfio_device_assign_container(struct vfio_device *device);
116−void vfio_device_unassign_container(struct vfio_device *device);
140+int vfio_group_use_container(struct vfio_group *group);
141+void vfio_group_unuse_container(struct vfio_group *group);
117142 int vfio_container_attach_group(struct vfio_container *container,
118143 struct vfio_group *group);
119144 void vfio_group_detach_container(struct vfio_group *group);
120145 void vfio_device_container_register(struct vfio_device *device);
121146 void vfio_device_container_unregister(struct vfio_device *device);
122−long vfio_container_ioctl_check_extension(struct vfio_container *container,
123− unsigned long arg);
147+int vfio_device_container_pin_pages(struct vfio_device *device,
148+ dma_addr_t iova, int npage,
149+ int prot, struct page **pages);
150+void vfio_device_container_unpin_pages(struct vfio_device *device,
151+ dma_addr_t iova, int npage);
152+int vfio_device_container_dma_rw(struct vfio_device *device,
153+ dma_addr_t iova, void *data,
154+ size_t len, bool write);
155+
124156 int __init vfio_container_init(void);
125157 void vfio_container_cleanup(void);
158+#else
159+static inline struct vfio_container *
160+vfio_container_from_file(struct file *filep)
161+{
162+ return NULL;
163+}
164+
165+static inline int vfio_group_use_container(struct vfio_group *group)
166+{
167+ return -EOPNOTSUPP;
168+}
169+
170+static inline void vfio_group_unuse_container(struct vfio_group *group)
171+{
172+}
173+
174+static inline int vfio_container_attach_group(struct vfio_container *container,
175+ struct vfio_group *group)
176+{
177+ return -EOPNOTSUPP;
178+}
179+
180+static inline void vfio_group_detach_container(struct vfio_group *group)
181+{
182+}
183+
184+static inline void vfio_device_container_register(struct vfio_device *device)
185+{
186+}
187+
188+static inline void vfio_device_container_unregister(struct vfio_device *device)
189+{
190+}
191+
192+static inline int vfio_device_container_pin_pages(struct vfio_device *device,
193+ dma_addr_t iova, int npage,
194+ int prot, struct page **pages)
195+{
196+ return -EOPNOTSUPP;
197+}
198+
199+static inline void vfio_device_container_unpin_pages(struct vfio_device *device,
200+ dma_addr_t iova, int npage)
201+{
202+}
203+
204+static inline int vfio_device_container_dma_rw(struct vfio_device *device,
205+ dma_addr_t iova, void *data,
206+ size_t len, bool write)
207+{
208+ return -EOPNOTSUPP;
209+}
210+
211+static inline int vfio_container_init(void)
212+{
213+ return 0;
214+}
215+static inline void vfio_container_cleanup(void)
216+{
217+}
218+#endif
219+
220+#if IS_ENABLED(CONFIG_IOMMUFD)
221+int vfio_iommufd_bind(struct vfio_device *device, struct iommufd_ctx *ictx);
222+void vfio_iommufd_unbind(struct vfio_device *device);
223+#else
224+static inline int vfio_iommufd_bind(struct vfio_device *device,
225+ struct iommufd_ctx *ictx)
226+{
227+ return -EOPNOTSUPP;
228+}
229+
230+static inline void vfio_iommufd_unbind(struct vfio_device *device)
231+{
232+}
233+#endif
126234
127235 #ifdef CONFIG_VFIO_NOIOMMU
128236 extern bool vfio_noiommu __read_mostly;
drivers/vfio/vfio_main.c+193−790
@@ -13,8 +13,6 @@ · 12 unmodified lines
1313 #include <linux/cdev.h>
1414 #include <linux/compat.h>
1515 #include <linux/device.h>
16−#include <linux/file.h>
17−#include <linux/anon_inodes.h>
1816 #include <linux/fs.h>
1917 #include <linux/idr.h>
2018 #include <linux/iommu.h>
@@ -35,6 +33,7 @@ · 14 unmodified lines
3533 #include <linux/pm_runtime.h>
3634 #include <linux/interval_tree.h>
3735 #include <linux/iova_bitmap.h>
36+#include <linux/iommufd.h>
3837 #include "vfio.h"
3938
4039 #define DRIVER_VERSION "0.3"
@@ -42,17 +41,11 @@ · 1 unmodified line
4241 #define DRIVER_DESC "VFIO - User Level meta-driver"
4342
4443 static struct vfio {
45− struct class *class;
46− struct list_head group_list;
47− struct mutex group_lock; /* locks group_list */
48− struct ida group_ida;
49− dev_t group_devt;
5044 struct class *device_class;
5145 struct ida device_ida;
5246 } vfio;
5347
5448 static DEFINE_XARRAY(vfio_device_set_xa);
55−static const struct file_operations vfio_group_fops;
5649
5750 int vfio_assign_device_set(struct vfio_device *device, void *set_id)
5851 {
@@ -138,208 +131,21 @@ unsigned int vfio_device_set_open_count(struct vfio_device_set *dev_set) · 79 unmodified lines
138131 }
139132 EXPORT_SYMBOL_GPL(vfio_device_set_open_count);
140133
141−/*
142− * Group objects - create, release, get, put, search
143− */
144−static struct vfio_group *
145−__vfio_group_get_from_iommu(struct iommu_group *iommu_group)
146−{
147− struct vfio_group *group;
148−
149− /*
150− * group->iommu_group from the vfio.group_list cannot be NULL
151− * under the vfio.group_lock.
152− */
153− list_for_each_entry(group, &vfio.group_list, vfio_next) {
154− if (group->iommu_group == iommu_group) {
155− refcount_inc(&group->drivers);
156− return group;
157− }
158− }
159− return NULL;
160−}
161−
162−static struct vfio_group *
163−vfio_group_get_from_iommu(struct iommu_group *iommu_group)
164−{
165− struct vfio_group *group;
166−
167− mutex_lock(&vfio.group_lock);
168− group = __vfio_group_get_from_iommu(iommu_group);
169− mutex_unlock(&vfio.group_lock);
170− return group;
171−}
172−
173−static void vfio_group_release(struct device *dev)
174−{
175− struct vfio_group *group = container_of(dev, struct vfio_group, dev);
176−
177− mutex_destroy(&group->device_lock);
178− mutex_destroy(&group->group_lock);
179− WARN_ON(group->iommu_group);
180− ida_free(&vfio.group_ida, MINOR(group->dev.devt));
181− kfree(group);
182−}
183−
184−static struct vfio_group *vfio_group_alloc(struct iommu_group *iommu_group,
185− enum vfio_group_type type)
186−{
187− struct vfio_group *group;
188− int minor;
189−
190− group = kzalloc(sizeof(*group), GFP_KERNEL);
191− if (!group)
192− return ERR_PTR(-ENOMEM);
193−
194− minor = ida_alloc_max(&vfio.group_ida, MINORMASK, GFP_KERNEL);
195− if (minor < 0) {
196− kfree(group);
197− return ERR_PTR(minor);
198− }
199−
200− device_initialize(&group->dev);
201− group->dev.devt = MKDEV(MAJOR(vfio.group_devt), minor);
202− group->dev.class = vfio.class;
203− group->dev.release = vfio_group_release;
204− cdev_init(&group->cdev, &vfio_group_fops);
205− group->cdev.owner = THIS_MODULE;
206−
207− refcount_set(&group->drivers, 1);
208− mutex_init(&group->group_lock);
209− INIT_LIST_HEAD(&group->device_list);
210− mutex_init(&group->device_lock);
211− group->iommu_group = iommu_group;
212− /* put in vfio_group_release() */
213− iommu_group_ref_get(iommu_group);
214− group->type = type;
215− BLOCKING_INIT_NOTIFIER_HEAD(&group->notifier);
216−
217− return group;
218−}
219−
220−static struct vfio_group *vfio_create_group(struct iommu_group *iommu_group,
221− enum vfio_group_type type)
222−{
223− struct vfio_group *group;
224− struct vfio_group *ret;
225− int err;
226−
227− group = vfio_group_alloc(iommu_group, type);
228− if (IS_ERR(group))
229− return group;
230−
231− err = dev_set_name(&group->dev, "%s%d",
232− group->type == VFIO_NO_IOMMU ? "noiommu-" : "",
233− iommu_group_id(iommu_group));
234− if (err) {
235− ret = ERR_PTR(err);
236− goto err_put;
237− }
238−
239− mutex_lock(&vfio.group_lock);
240−
241− /* Did we race creating this group? */
242− ret = __vfio_group_get_from_iommu(iommu_group);
243− if (ret)
244− goto err_unlock;
245−
246− err = cdev_device_add(&group->cdev, &group->dev);
247− if (err) {
248− ret = ERR_PTR(err);
249− goto err_unlock;
250− }
251−
252− list_add(&group->vfio_next, &vfio.group_list);
253−
254− mutex_unlock(&vfio.group_lock);
255− return group;
256−
257−err_unlock:
258− mutex_unlock(&vfio.group_lock);
259−err_put:
260− put_device(&group->dev);
261− return ret;
262−}
263−
264−static void vfio_device_remove_group(struct vfio_device *device)
265−{
266− struct vfio_group *group = device->group;
267− struct iommu_group *iommu_group;
268−
269− if (group->type == VFIO_NO_IOMMU || group->type == VFIO_EMULATED_IOMMU)
270− iommu_group_remove_device(device->dev);
271−
272− /* Pairs with vfio_create_group() / vfio_group_get_from_iommu() */
273− if (!refcount_dec_and_mutex_lock(&group->drivers, &vfio.group_lock))
274− return;
275− list_del(&group->vfio_next);
276−
277− /*
278− * We could concurrently probe another driver in the group that might
279− * race vfio_device_remove_group() with vfio_get_group(), so we have to
280− * ensure that the sysfs is all cleaned up under lock otherwise the
281− * cdev_device_add() will fail due to the name aready existing.
282− */
283− cdev_device_del(&group->cdev, &group->dev);
284−
285− mutex_lock(&group->group_lock);
286− /*
287− * These data structures all have paired operations that can only be
288− * undone when the caller holds a live reference on the device. Since
289− * all pairs must be undone these WARN_ON's indicate some caller did not
290− * properly hold the group reference.
291− */
292− WARN_ON(!list_empty(&group->device_list));
293− WARN_ON(group->notifier.head);
294−
295− /*
296− * Revoke all users of group->iommu_group. At this point we know there
297− * are no devices active because we are unplugging the last one. Setting
298− * iommu_group to NULL blocks all new users.
299− */
300− if (group->container)
301− vfio_group_detach_container(group);
302− iommu_group = group->iommu_group;
303− group->iommu_group = NULL;
304− mutex_unlock(&group->group_lock);
305− mutex_unlock(&vfio.group_lock);
306−
307− iommu_group_put(iommu_group);
308− put_device(&group->dev);
309−}
310−
311134 /*
312135 * Device objects - create, release, get, put, search
313136 */
314137 /* Device reference always implies a group reference */
315−static void vfio_device_put_registration(struct vfio_device *device)
138+void vfio_device_put_registration(struct vfio_device *device)
316139 {
317140 if (refcount_dec_and_test(&device->refcount))
318141 complete(&device->comp);
319142 }
320143
321−static bool vfio_device_try_get_registration(struct vfio_device *device)
144+bool vfio_device_try_get_registration(struct vfio_device *device)
322145 {
323146 return refcount_inc_not_zero(&device->refcount);
324147 }
325148
326−static struct vfio_device *vfio_group_get_device(struct vfio_group *group,
327− struct device *dev)
328−{
329− struct vfio_device *device;
330−
331− mutex_lock(&group->device_lock);
332− list_for_each_entry(device, &group->device_list, group_next) {
333− if (device->dev == dev &&
334− vfio_device_try_get_registration(device)) {
335− mutex_unlock(&group->device_lock);
336− return device;
337− }
338− }
339− mutex_unlock(&group->device_lock);
340− return NULL;
341−}
342−
343149 /*
344150 * VFIO driver API
345151 */
@@ -448,94 +254,15 @@ void vfio_free_device(struct vfio_device *device) · 102 unmodified lines
448254 }
449255 EXPORT_SYMBOL_GPL(vfio_free_device);
450256
451−static struct vfio_group *vfio_noiommu_group_alloc(struct device *dev,
452− enum vfio_group_type type)
453−{
454− struct iommu_group *iommu_group;
455− struct vfio_group *group;
456− int ret;
457−
458− iommu_group = iommu_group_alloc();
459− if (IS_ERR(iommu_group))
460− return ERR_CAST(iommu_group);
461−
462− ret = iommu_group_set_name(iommu_group, "vfio-noiommu");
463− if (ret)
464− goto out_put_group;
465− ret = iommu_group_add_device(iommu_group, dev);
466− if (ret)
467− goto out_put_group;
468−
469− group = vfio_create_group(iommu_group, type);
470− if (IS_ERR(group)) {
471− ret = PTR_ERR(group);
472− goto out_remove_device;
473− }
474− iommu_group_put(iommu_group);
475− return group;
476−
477−out_remove_device:
478− iommu_group_remove_device(dev);
479−out_put_group:
480− iommu_group_put(iommu_group);
481− return ERR_PTR(ret);
482−}
483−
484−static struct vfio_group *vfio_group_find_or_alloc(struct device *dev)
485−{
486− struct iommu_group *iommu_group;
487− struct vfio_group *group;
488−
489− iommu_group = iommu_group_get(dev);
490− if (!iommu_group && vfio_noiommu) {
491− /*
492− * With noiommu enabled, create an IOMMU group for devices that
493− * don't already have one, implying no IOMMU hardware/driver
494− * exists. Taint the kernel because we're about to give a DMA
495− * capable device to a user without IOMMU protection.
496− */
497− group = vfio_noiommu_group_alloc(dev, VFIO_NO_IOMMU);
498− if (!IS_ERR(group)) {
499− add_taint(TAINT_USER, LOCKDEP_STILL_OK);
500− dev_warn(dev, "Adding kernel taint for vfio-noiommu group on device\n");
501− }
502− return group;
503− }
504−
505− if (!iommu_group)
506− return ERR_PTR(-EINVAL);
507−
508− /*
509− * VFIO always sets IOMMU_CACHE because we offer no way for userspace to
510− * restore cache coherency. It has to be checked here because it is only
511− * valid for cases where we are using iommu groups.
512− */
513− if (!device_iommu_capable(dev, IOMMU_CAP_CACHE_COHERENCY)) {
514− iommu_group_put(iommu_group);
515− return ERR_PTR(-EINVAL);
516− }
517−
518− group = vfio_group_get_from_iommu(iommu_group);
519− if (!group)
520− group = vfio_create_group(iommu_group, VFIO_IOMMU);
521−
522− /* The vfio_group holds a reference to the iommu_group */
523− iommu_group_put(iommu_group);
524− return group;
525−}
526−
527257 static int __vfio_register_dev(struct vfio_device *device,
528− struct vfio_group *group)
258+ enum vfio_group_type type)
529259 {
530− struct vfio_device *existing_device;
531260 int ret;
532261
533− /*
534− * In all cases group is the output of one of the group allocation
535− * functions and we have group->drivers incremented for us.
536− */
537− if (IS_ERR(group))
538− return PTR_ERR(group);
262+ if (WARN_ON(device->ops->bind_iommufd &&
263+ (!device->ops->unbind_iommufd ||
264+ !device->ops->attach_ioas)))
265+ return -EINVAL;
539266
540267 /*
541268 * If the driver doesn't specify a set then the device is added to a
@@ -544,25 +271,13 @@ static int __vfio_register_dev(struct vfio_device *device, · 2 unmodified lines
544271 if (!device->dev_set)
545272 vfio_assign_device_set(device, device);
546273
547− existing_device = vfio_group_get_device(group, device->dev);
548− if (existing_device) {
549− /*
550− * group->iommu_group is non-NULL because we hold the drivers
551− * refcount.
552− */
553− dev_WARN(device->dev, "Device already exists on group %d\n",
554− iommu_group_id(group->iommu_group));
555− vfio_device_put_registration(existing_device);
556− ret = -EBUSY;
557− goto err_out;
558− }
559−
560− /* Our reference on group is moved to the device */
561− device->group = group;
562−
563274 ret = dev_set_name(&device->device, "vfio%d", device->index);
564275 if (ret)
565− goto err_out;
276+ return ret;
277+
278+ ret = vfio_device_set_group(device, type);
279+ if (ret)
280+ return ret;
566281
567282 ret = device_add(&device->device);
568283 if (ret)
@@ -571,9 +286,7 @@ static int __vfio_register_dev(struct vfio_device *device, · 2 unmodified lines
571286 /* Refcounting can't start until the driver calls register */
572287 refcount_set(&device->refcount, 1);
573288
574− mutex_lock(&group->device_lock);
575− list_add(&device->group_next, &group->device_list);
576− mutex_unlock(&group->device_lock);
289+ vfio_device_group_register(device);
577290
578291 return 0;
579292 err_out:
@@ -583,8 +296,7 @@ err_out: · 3 unmodified lines
583296
584297 int vfio_register_group_dev(struct vfio_device *device)
585298 {
586− return __vfio_register_dev(device,
587− vfio_group_find_or_alloc(device->dev));
299+ return __vfio_register_dev(device, VFIO_IOMMU);
588300 }
589301 EXPORT_SYMBOL_GPL(vfio_register_group_dev);
590302
@@ -594,46 +306,15 @@ EXPORT_SYMBOL_GPL(vfio_register_group_dev); · 3 unmodified lines
594306 */
595307 int vfio_register_emulated_iommu_dev(struct vfio_device *device)
596308 {
597− return __vfio_register_dev(device,
598− vfio_noiommu_group_alloc(device->dev, VFIO_EMULATED_IOMMU));
309+ return __vfio_register_dev(device, VFIO_EMULATED_IOMMU);
599310 }
600311 EXPORT_SYMBOL_GPL(vfio_register_emulated_iommu_dev);
601312
602−static struct vfio_device *vfio_device_get_from_name(struct vfio_group *group,
603− char *buf)
604−{
605− struct vfio_device *it, *device = ERR_PTR(-ENODEV);
606−
607− mutex_lock(&group->device_lock);
608− list_for_each_entry(it, &group->device_list, group_next) {
609− int ret;
610−
611− if (it->ops->match) {
612− ret = it->ops->match(it, buf);
613− if (ret < 0) {
614− device = ERR_PTR(ret);
615− break;
616− }
617− } else {
618− ret = !strcmp(dev_name(it->dev), buf);
619− }
620−
621− if (ret && vfio_device_try_get_registration(it)) {
622− device = it;
623− break;
624− }
625− }
626− mutex_unlock(&group->device_lock);
627−
628− return device;
629−}
630−
631313 /*
632314 * Decrement the device reference count and wait for the device to be
633315 * removed. Open file descriptors for the device... */
634316 void vfio_unregister_group_dev(struct vfio_device *device)
635317 {
636− struct vfio_group *group = device->group;
637318 unsigned int i = 0;
638319 bool interrupted = false;
639320 long rc;
@@ -661,333 +342,101 @@ void vfio_unregister_group_dev(struct vfio_device *device) · 21 unmodified lines
661342 }
662343 }
663344
664− mutex_lock(&group->device_lock);
665− list_del(&device->group_next);
666− mutex_unlock(&group->device_lock);
345+ vfio_device_group_unregister(device);
667346
668347 /* Balances device_add in register path */
669348 device_del(&device->device);
670349
350+ /* Balances vfio_device_set_group in register path */
671351 vfio_device_remove_group(device);
672352 }
673353 EXPORT_SYMBOL_GPL(vfio_unregister_group_dev);
674354
675−/*
676− * VFIO Group fd, /dev/vfio/$GROUP
677− */
678−/*
679− * VFIO_GROUP_UNSET_CONTAINER should fail if there are other users or
680− * if there was no container to unset. Since the ioctl is called on
681− * the group, we know that still exists, therefore the only valid
682− * transition here is 1->0.
683− */
684−static int vfio_group_ioctl_unset_container(struct vfio_group *group)
355+/* true if the vfio_device has open_device() called but not close_device() */
356+static bool vfio_assert_device_open(struct vfio_device *device)
685357 {
686− int ret = 0;
687−
688− mutex_lock(&group->group_lock);
689− if (!group->container) {
690− ret = -EINVAL;
691− goto out_unlock;
692− }
693− if (group->container_users != 1) {
694− ret = -EBUSY;
695− goto out_unlock;
696− }
697− vfio_group_detach_container(group);
698−
699−out_unlock:
700− mutex_unlock(&group->group_lock);
701− return ret;
358+ return !WARN_ON_ONCE(!READ_ONCE(device->open_count));
702359 }
703360
704−static int vfio_group_ioctl_set_container(struct vfio_group *group,
705− int __user *arg)
361+static int vfio_device_first_open(struct vfio_device *device,
362+ struct iommufd_ctx *iommufd, struct kvm *kvm)
706363 {
707− struct vfio_container *container;
708− struct fd f;
709364 int ret;
710− int fd;
711365
712− if (get_user(fd, arg))
713− return -EFAULT;
366+ lockdep_assert_held(&device->dev_set->lock);
714367
715− f = fdget(fd);
716− if (!f.file)
717− return -EBADF;
368+ if (!try_module_get(device->dev->driver->owner))
369+ return -ENODEV;
718370
719− mutex_lock(&group->group_lock);
720− if (group->container || WARN_ON(group->container_users)) {
721− ret = -EINVAL;
722− goto out_unlock;
723− }
724− if (!group->iommu_group) {
725− ret = -ENODEV;
726− goto out_unlock;
727− }
371+ if (iommufd)
372+ ret = vfio_iommufd_bind(device, iommufd);
373+ else
374+ ret = vfio_device_group_use_iommu(device);
375+ if (ret)
376+ goto err_module_put;
728377
729− container = vfio_container_from_file(f.file);
730− ret = -EINVAL;
731− if (container) {
732− ret = vfio_container_attach_group(container, group);
733− goto out_unlock;
378+ device->kvm = kvm;
379+ if (device->ops->open_device) {
380+ ret = device->ops->open_device(device);
381+ if (ret)
382+ goto err_unuse_iommu;
734383 }
384+ return 0;
735385
736−out_unlock:
737− mutex_unlock(&group->group_lock);
738− fdput(f);
386+err_unuse_iommu:
387+ device->kvm = NULL;
388+ if (iommufd)
389+ vfio_iommufd_unbind(device);
390+ else
391+ vfio_device_group_unuse_iommu(device);
392+err_module_put:
393+ module_put(device->dev->driver->owner);
739394 return ret;
740395 }
741396
742−static const struct file_operations vfio_device_fops;
743−
744−/* true if the vfio_device has open_device() called but not close_device() */
745−bool vfio_assert_device_open(struct vfio_device *device)
397+static void vfio_device_last_close(struct vfio_device *device,
398+ struct iommufd_ctx *iommufd)
746399 {
747− return !WARN_ON_ONCE(!READ_ONCE(device->open_count));
400+ lockdep_assert_held(&device->dev_set->lock);
401+
402+ if (device->ops->close_device)
403+ device->ops->close_device(device);
404+ device->kvm = NULL;
405+ if (iommufd)
406+ vfio_iommufd_unbind(device);
407+ else
408+ vfio_device_group_unuse_iommu(device);
409+ module_put(device->dev->driver->owner);
748410 }
749411
750−static struct file *vfio_device_open(struct vfio_device *device)
412+int vfio_device_open(struct vfio_device *device,
413+ struct iommufd_ctx *iommufd, struct kvm *kvm)
751414 {
752− struct file *filep;
753− int ret;
754−
755− mutex_lock(&device->group->group_lock);
756− ret = vfio_device_assign_container(device);
757− mutex_unlock(&device->group->group_lock);
758− if (ret)
759− return ERR_PTR(ret);
760−
761− if (!try_module_get(device->dev->driver->owner)) {
762− ret = -ENODEV;
763− goto err_unassign_container;
764− }
415+ int ret = 0;
765416
766417 mutex_lock(&device->dev_set->lock);
767418 device->open_count++;
768419 if (device->open_count == 1) {
769− /*
770− * Here we pass the KVM pointer with the group under the read
771− * lock. If the device driver will use it, it must obtain a
772− * reference and release it during close_device.
773− */
774− mutex_lock(&device->group->group_lock);
775− device->kvm = device->group->kvm;
776−
777− if (device->ops->open_device) {
778− ret = device->ops->open_device(device);
779− if (ret)
780− goto err_undo_count;
781− }
782− vfio_device_container_register(device);
783− mutex_unlock(&device->group->group_lock);
784− }
785− mutex_unlock(&device->dev_set->lock);
786−
787− /*
788− * We can't use anon_inode_getfd() because we need to modify
789− * the f_mode flags directly to allow more than just ioctls
790− */
791− filep = anon_inode_getfile("[vfio-device]", &vfio_device_fops,
792− device, O_RDWR);
793− if (IS_ERR(filep)) {
794− ret = PTR_ERR(filep);
795− goto err_close_device;
796− }
797−
798− /*
799− * TODO: add an anon_inode interface to do this.
800− * Appears to be missing by lack of need rather than
801− * explicitly prevented. Now there's need.
802− */
803− filep->f_mode |= (FMODE_PREAD | FMODE_PWRITE);
804−
805− if (device->group->type == VFIO_NO_IOMMU)
806− dev_warn(device->dev, "vfio-noiommu device opened by user "
807− "(%s:%d)\n", current->comm, task_pid_nr(current));
808− /*
809− * On success the ref of device is moved to the file and
810− * put in vfio_device_fops_release()
811− */
812− return filep;
813−
814−err_close_device:
815− mutex_lock(&device->dev_set->lock);
816− mutex_lock(&device->group->group_lock);
817− if (device->open_count == 1) {
818− if (device->ops->close_device)
819− device->ops->close_device(device);
820−
821− vfio_device_container_unregister(device);
420+ ret = vfio_device_first_open(device, iommufd, kvm);
421+ if (ret)
422+ device->open_count--;
822423 }
823−err_undo_count:
824− mutex_unlock(&device->group->group_lock);
825− device->open_count--;
826− if (device->open_count == 0 && device->kvm)
827− device->kvm = NULL;
828424 mutex_unlock(&device->dev_set->lock);
829− module_put(device->dev->driver->owner);
830−err_unassign_container:
831− vfio_device_unassign_container(device);
832− return ERR_PTR(ret);
833−}
834−
835−static int vfio_group_ioctl_get_device_fd(struct vfio_group *group,
836− char __user *arg)
837−{
838− struct vfio_device *device;
839− struct file *filep;
840− char *buf;
841− int fdno;
842− int ret;
843−
844− buf = strndup_user(arg, PAGE_SIZE);
845− if (IS_ERR(buf))
846− return PTR_ERR(buf);
847−
848− device = vfio_device_get_from_name(group, buf);
849− kfree(buf);
850− if (IS_ERR(device))
851− return PTR_ERR(device);
852−
853− fdno = get_unused_fd_flags(O_CLOEXEC);
854− if (fdno < 0) {
855− ret = fdno;
856− goto err_put_device;
857− }
858−
859− filep = vfio_device_open(device);
860− if (IS_ERR(filep)) {
861− ret = PTR_ERR(filep);
862− goto err_put_fdno;
863− }
864−
865− fd_install(fdno, filep);
866− return fdno;
867−
868−err_put_fdno:
869− put_unused_fd(fdno);
870−err_put_device:
871− vfio_device_put_registration(device);
872− return ret;
873−}
874−
875−static int vfio_group_ioctl_get_status(struct vfio_group *group,
876− struct vfio_group_status __user *arg)
877−{
878− unsigned long minsz = offsetofend(struct vfio_group_status, flags);
879− struct vfio_group_status status;
880−
881− if (copy_from_user(&status, arg, minsz))
882− return -EFAULT;
883−
884− if (status.argsz < minsz)
885− return -EINVAL;
886−
887− status.flags = 0;
888−
889− mutex_lock(&group->group_lock);
890− if (!group->iommu_group) {
891− mutex_unlock(&group->group_lock);
892− return -ENODEV;
893− }
894−
895− if (group->container)
896− status.flags |= VFIO_GROUP_FLAGS_CONTAINER_SET |
897− VFIO_GROUP_FLAGS_VIABLE;
898− else if (!iommu_group_dma_owner_claimed(group->iommu_group))
899− status.flags |= VFIO_GROUP_FLAGS_VIABLE;
900− mutex_unlock(&group->group_lock);
901−
902− if (copy_to_user(arg, &status, minsz))
903− return -EFAULT;
904− return 0;
905−}
906−
907−static long vfio_group_fops_unl_ioctl(struct file *filep,
908− unsigned int cmd, unsigned long arg)
909−{
910− struct vfio_group *group = filep->private_data;
911− void __user *uarg = (void __user *)arg;
912−
913− switch (cmd) {
914− case VFIO_GROUP_GET_DEVICE_FD:
915− return vfio_group_ioctl_get_device_fd(group, uarg);
916− case VFIO_GROUP_GET_STATUS:
917− return vfio_group_ioctl_get_status(group, uarg);
918− case VFIO_GROUP_SET_CONTAINER:
919− return vfio_group_ioctl_set_container(group, uarg);
920− case VFIO_GROUP_UNSET_CONTAINER:
921− return vfio_group_ioctl_unset_container(group);
922− default:
923− return -ENOTTY;
924− }
925−}
926−
927−static int vfio_group_fops_open(struct inode *inode, struct file *filep)
928−{
929− struct vfio_group *group =
930− container_of(inode->i_cdev, struct vfio_group, cdev);
931− int ret;
932−
933− mutex_lock(&group->group_lock);
934425
935− /*
936− * drivers can be zero if this races with vfio_device_remove_group(), it
937− * will be stable at 0 under the group rwsem
938− */
939− if (refcount_read(&group->drivers) == 0) {
940− ret = -ENODEV;
941− goto out_unlock;
942− }
943−
944− if (group->type == VFIO_NO_IOMMU && !capable(CAP_SYS_RAWIO)) {
945− ret = -EPERM;
946− goto out_unlock;
947− }
948−
949− /*
950− * Do we need multiple instances of the group open? Seems not.
951− */
952− if (group->opened_file) {
953− ret = -EBUSY;
954− goto out_unlock;
955− }
956− group->opened_file = filep;
957− filep->private_data = group;
958− ret = 0;
959−out_unlock:
960− mutex_unlock(&group->group_lock);
961426 return ret;
962427 }
963428
964−static int vfio_group_fops_release(struct inode *inode, struct file *filep)
429+void vfio_device_close(struct vfio_device *device,
430+ struct iommufd_ctx *iommufd)
965431 {
966− struct vfio_group *group = filep->private_data;
967−
968− filep->private_data = NULL;
969−
970− mutex_lock(&group->group_lock);
971− /*
972− * Device FDs hold a group file reference, therefore the group release
973− * is only called when there are no open devices.
974− */
975− WARN_ON(group->notifier.head);
976− if (group->container)
977− vfio_group_detach_container(group);
978− group->opened_file = NULL;
979− mutex_unlock(&group->group_lock);
980− return 0;
432+ mutex_lock(&device->dev_set->lock);
433+ vfio_assert_device_open(device);
434+ if (device->open_count == 1)
435+ vfio_device_last_close(device, iommufd);
436+ device->open_count--;
437+ mutex_unlock(&device->dev_set->lock);
981438 }
982439
983−static const struct file_operations vfio_group_fops = {
984− .owner = THIS_MODULE,
985− .unlocked_ioctl = vfio_group_fops_unl_ioctl,
986− .compat_ioctl = compat_ptr_ioctl,
987− .open = vfio_group_fops_open,
988− .release = vfio_group_fops_release,
989−};
990−
991440 /*
992441 * Wrapper around pm_runtime_resume_and_get().
993442 * Return error code on failure or 0 on success.
@@ -1028,24 +477,7 @@ static int vfio_device_fops_release(struct inode *inode, struct file *filep) · 34 unmodified lines
1028477 {
1029478 struct vfio_device *device = filep->private_data;
1030479
1031− mutex_lock(&device->dev_set->lock);
1032− vfio_assert_device_open(device);
1033− mutex_lock(&device->group->group_lock);
1034− if (device->open_count == 1) {
1035− if (device->ops->close_device)
1036− device->ops->close_device(device);
1037−
1038− vfio_device_container_unregister(device);
1039− }
1040− mutex_unlock(&device->group->group_lock);
1041− device->open_count--;
1042− if (device->open_count == 0)
1043− device->kvm = NULL;
1044− mutex_unlock(&device->dev_set->lock);
1045−
1046− module_put(device->dev->driver->owner);
1047−
1048− vfio_device_unassign_container(device);
480+ vfio_device_group_close(device);
1049481
1050482 vfio_device_put_registration(device);
1051483
@@ -1568,7 +1000,7 @@ static int vfio_device_fops_mmap(struct file *filep, struct vm_area_struct *vma) · 516 unmodified lines
15681000 return device->ops->mmap(device, vma);
15691001 }
15701002
1571−static const struct file_operations vfio_device_fops = {
1003+const struct file_operations vfio_device_fops = {
15721004 .owner = THIS_MODULE,
15731005 .release = vfio_device_fops_release,
15741006 .read = vfio_device_fops_read,
@@ -1578,118 +1010,6 @@ static const struct file_operations vfio_device_fops = { · 3 unmodified lines
15781010 .mmap = vfio_device_fops_mmap,
15791011 };
15801012
1581−/**
1582− * vfio_file_iommu_group - Return the struct iommu_group for the vfio group file
1583− * @file: VFIO group file
1584− *
1585− * The returned iommu_group is valid as long as a ref is held on the file. This
1586− * returns a reference on the group. This function is deprecated, only the SPAPR
1587− * path in kvm should call it.
1588− */
1589−struct iommu_group *vfio_file_iommu_group(struct file *file)
1590−{
1591− struct vfio_group *group = file->private_data;
1592− struct iommu_group *iommu_group = NULL;
1593−
1594− if (!IS_ENABLED(CONFIG_SPAPR_TCE_IOMMU))
1595− return NULL;
1596−
1597− if (!vfio_file_is_group(file))
1598− return NULL;
1599−
1600− mutex_lock(&group->group_lock);
1601− if (group->iommu_group) {
1602− iommu_group = group->iommu_group;
1603− iommu_group_ref_get(iommu_group);
1604− }
1605− mutex_unlock(&group->group_lock);
1606− return iommu_group;
1607−}
1608−EXPORT_SYMBOL_GPL(vfio_file_iommu_group);
1609−
1610−/**
1611− * vfio_file_is_group - True if the file is usable with VFIO aPIS
1612− * @file: VFIO group file
1613− */
1614−bool vfio_file_is_group(struct file *file)
1615−{
1616− return file->f_op == &vfio_group_fops;
1617−}
1618−EXPORT_SYMBOL_GPL(vfio_file_is_group);
1619−
1620−/**
1621− * vfio_file_enforced_coherent - True if the DMA associated with the VFIO file
1622− * is always CPU cache coherent
1623− * @file: VFIO group file
1624− *
1625− * Enforced coherency means that the IOMMU ignores things like the PCIe no-snoop
1626− * bit in DMA transactions. A return of false indicates that the user has
1627− * rights to access additional instructions such as wbinvd on x86.
1628− */
1629−bool vfio_file_enforced_coherent(struct file *file)
1630−{
1631− struct vfio_group *group = file->private_data;
1632− bool ret;
1633−
1634− if (!vfio_file_is_group(file))
1635− return true;
1636−
1637− mutex_lock(&group->group_lock);
1638− if (group->container) {
1639− ret = vfio_container_ioctl_check_extension(group->container,
1640− VFIO_DMA_CC_IOMMU);
1641− } else {
1642− /*
1643− * Since the coherency state is determined only once a container
1644− * is attached the user must do so before they can prove they
1645− * have permission.
1646− */
1647− ret = true;
1648− }
1649− mutex_unlock(&group->group_lock);
1650− return ret;
1651−}
1652−EXPORT_SYMBOL_GPL(vfio_file_enforced_coherent);
1653−
1654−/**
1655− * vfio_file_set_kvm - Link a kvm with VFIO drivers
1656− * @file: VFIO group file
1657− * @kvm: KVM to link
1658− *
1659− * When a VFIO device is first opened the KVM will be available in
1660− * device->kvm if one was associated with the group.
1661− */
1662−void vfio_file_set_kvm(struct file *file, struct kvm *kvm)
1663−{
1664− struct vfio_group *group = file->private_data;
1665−
1666− if (!vfio_file_is_group(file))
1667− return;
1668−
1669− mutex_lock(&group->group_lock);
1670− group->kvm = kvm;
1671− mutex_unlock(&group->group_lock);
1672−}
1673−EXPORT_SYMBOL_GPL(vfio_file_set_kvm);
1674−
1675−/**
1676− * vfio_file_has_dev - True if the VFIO file is a handle for device
1677− * @file: VFIO file to check
1678− * @device: Device that must be part of the file
1679− *
1680− * Returns true if given file has permission to manipulate the given device.
1681− */
1682−bool vfio_file_has_dev(struct file *file, struct vfio_device *device)
1683−{
1684− struct vfio_group *group = file->private_data;
1685−
1686− if (!vfio_file_is_group(file))
1687− return false;
1688−
1689− return group == device->group;
1690−}
1691−EXPORT_SYMBOL_GPL(vfio_file_has_dev);
1692−
16931013 /*
16941014 * Sub-module support
16951015 */
@@ -1810,35 +1130,136 @@ int vfio_set_irqs_validate_and_prepare(struct vfio_irq_set *hdr, int num_irqs, · 114 unmodified lines
18101130 EXPORT_SYMBOL(vfio_set_irqs_validate_and_prepare);
18111131
18121132 /*
1813− * Module/class support
1133+ * Pin contiguous user pages and return their associated host pages for local
1134+ * domain only.
1135+ * @device [in] : device
1136+ * @iova [in] : starting IOVA of user pages to be pinned.
1137+ * @npage [in] : count of pages to be pinned. This count should not
1138+ * be greater than VFIO_PIN_PAGES_MAX_ENTRIES.
1139+ * @prot [in] : protection flags
1140+ * @pages[out] : array of host pages
1141+ * Return error or number of pages pinned.
1142+ *
1143+ * A driver may only call this function if the vfio_device was created
1144+ * by vfio_register_emulated_iommu_dev() due to vfio_device_container_pin_pages().
1145+ */
1146+int vfio_pin_pages(struct vfio_device *device, dma_addr_t iova,
1147+ int npage, int prot, struct page **pages)
1148+{
1149+ /* group->container cannot change while a vfio device is open */
1150+ if (!pages || !npage || WARN_ON(!vfio_assert_device_open(device)))
1151+ return -EINVAL;
1152+ if (vfio_device_has_container(device))
1153+ return vfio_device_container_pin_pages(device, iova,
1154+ npage, prot, pages);
1155+ if (device->iommufd_access) {
1156+ int ret;
1157+
1158+ if (iova > ULONG_MAX)
1159+ return -EINVAL;
1160+ /*
1161+ * VFIO ignores the sub page offset, npages is from the start of
1162+ * a PAGE_SIZE chunk of IOVA. The caller is expected to recover
1163+ * the sub page offset by doing:
1164+ * pages[0] + (iova % PAGE_SIZE)
1165+ */
1166+ ret = iommufd_access_pin_pages(
1167+ device->iommufd_access, ALIGN_DOWN(iova, PAGE_SIZE),
1168+ npage * PAGE_SIZE, pages,
1169+ (prot & IOMMU_WRITE) ? IOMMUFD_ACCESS_RW_WRITE : 0);
1170+ if (ret)
1171+ return ret;
1172+ return npage;
1173+ }
1174+ return -EINVAL;
1175+}
1176+EXPORT_SYMBOL(vfio_pin_pages);
1177+
1178+/*
1179+ * Unpin contiguous host pages for local domain only.
1180+ * @device [in] : device
1181+ * @iova [in] : starting address of user pages to be unpinned.
1182+ * @npage [in] : count of pages to be unpinned. This count should not
1183+ * be greater than VFIO_PIN_PAGES_MAX_ENTRIES.
1184+ */
1185+void vfio_unpin_pages(struct vfio_device *device, dma_addr_t iova, int npage)
1186+{
1187+ if (WARN_ON(!vfio_assert_device_open(device)))
1188+ return;
1189+
1190+ if (vfio_device_has_container(device)) {
1191+ vfio_device_container_unpin_pages(device, iova, npage);
1192+ return;
1193+ }
1194+ if (device->iommufd_access) {
1195+ if (WARN_ON(iova > ULONG_MAX))
1196+ return;
1197+ iommufd_access_unpin_pages(device->iommufd_access,
1198+ ALIGN_DOWN(iova, PAGE_SIZE),
1199+ npage * PAGE_SIZE);
1200+ return;
1201+ }
1202+}
1203+EXPORT_SYMBOL(vfio_unpin_pages);
1204+
1205+/*
1206+ * This interface allows the CPUs to perform some sort of virtual DMA on
1207+ * behalf of the device.
1208+ *
1209+ * CPUs read/write from/into a range of IOVAs pointing to user space memory
1210+ * into/from a kernel buffer.
1211+ *
1212+ * As the read/write of user space memory is conducted via the CPUs and is
1213+ * not a real device DMA, it is not necessary to pin the user space memory.
1214+ *
1215+ * @device [in] : VFIO device
1216+ * @iova [in] : base IOVA of a user space buffer
1217+ * @data [in] : pointer to kernel buffer
1218+ * @len [in] : kernel buffer length
1219+ * @write : indicate read or write
1220+ * Return error code on failure or 0 on success.
18141221 */
1815−static char *vfio_devnode(struct device *dev, umode_t *mode)
1222+int vfio_dma_rw(struct vfio_device *device, dma_addr_t iova, void *data,
1223+ size_t len, bool write)
18161224 {
1817− return kasprintf(GFP_KERNEL, "vfio/%s", dev_name(dev));
1225+ if (!data || len <= 0 || !vfio_assert_device_open(device))
1226+ return -EINVAL;
1227+
1228+ if (vfio_device_has_container(device))
1229+ return vfio_device_container_dma_rw(device, iova,
1230+ data, len, write);
1231+
1232+ if (device->iommufd_access) {
1233+ unsigned int flags = 0;
1234+
1235+ if (iova > ULONG_MAX)
1236+ return -EINVAL;
1237+
1238+ /* VFIO historically tries to auto-detect a kthread */
1239+ if (!current->mm)
1240+ flags |= IOMMUFD_ACCESS_RW_KTHREAD;
1241+ if (write)
1242+ flags |= IOMMUFD_ACCESS_RW_WRITE;
1243+ return iommufd_access_rw(device->iommufd_access, iova, data,
1244+ len, flags);
1245+ }
1246+ return -EINVAL;
18181247 }
1248+EXPORT_SYMBOL(vfio_dma_rw);
18191249
1250+/*
1251+ * Module/class support
1252+ */
18201253 static int __init vfio_init(void)
18211254 {
18221255 int ret;
18231256
1824− ida_init(&vfio.group_ida);
18251257 ida_init(&vfio.device_ida);
1826− mutex_init(&vfio.group_lock);
1827− INIT_LIST_HEAD(&vfio.group_list);
18281258
1829− ret = vfio_container_init();
1259+ ret = vfio_group_init();
18301260 if (ret)
18311261 return ret;
18321262
1833− /* /dev/vfio/$GROUP */
1834− vfio.class = class_create(THIS_MODULE, "vfio");
1835− if (IS_ERR(vfio.class)) {
1836− ret = PTR_ERR(vfio.class);
1837− goto err_group_class;
1838− }
1839−
1840− vfio.class->devnode = vfio_devnode;
1841−
18421263 /* /sys/class/vfio-dev/vfioX */
18431264 vfio.device_class = class_create(THIS_MODULE, "vfio-dev");
18441265 if (IS_ERR(vfio.device_class)) {
@@ -1846,36 +1267,20 @@ static int __init vfio_init(void) · 1 unmodified line
18461267 goto err_dev_class;
18471268 }
18481269
1849− ret = alloc_chrdev_region(&vfio.group_devt, 0, MINORMASK + 1, "vfio");
1850− if (ret)
1851− goto err_alloc_chrdev;
1852−
18531270 pr_info(DRIVER_DESC " version: " DRIVER_VERSION "\n");
18541271 return 0;
18551272
1856−err_alloc_chrdev:
1857− class_destroy(vfio.device_class);
1858− vfio.device_class = NULL;
18591273 err_dev_class:
1860− class_destroy(vfio.class);
1861− vfio.class = NULL;
1862−err_group_class:
1863− vfio_container_cleanup();
1274+ vfio_group_cleanup();
18641275 return ret;
18651276 }
18661277
18671278 static void __exit vfio_cleanup(void)
18681279 {
1869− WARN_ON(!list_empty(&vfio.group_list));
1870−
18711280 ida_destroy(&vfio.device_ida);
1872− ida_destroy(&vfio.group_ida);
1873− unregister_chrdev_region(vfio.group_devt, MINORMASK + 1);
18741281 class_destroy(vfio.device_class);
18751282 vfio.device_class = NULL;
1876− class_destroy(vfio.class);
1877− vfio_container_cleanup();
1878− vfio.class = NULL;
1283+ vfio_group_cleanup();
18791284 xa_destroy(&vfio_device_set_xa);
18801285 }
18811286
@@ -1886,6 +1291,4 @@ MODULE_VERSION(DRIVER_VERSION); · 4 unmodified lines
18861291 MODULE_LICENSE("GPL v2");
18871292 MODULE_AUTHOR(DRIVER_AUTHOR);
18881293 MODULE_DESCRIPTION(DRIVER_DESC);
1889−MODULE_ALIAS_MISCDEV(VFIO_MINOR);
1890−MODULE_ALIAS("devname:vfio/vfio");
18911294 MODULE_SOFTDEP("post: vfio_iommu_type1 vfio_iommu_spapr_tce");
include/linux/intel-svm.h+0−13
@@ -13,17 +13,4 @@ · 12 unmodified lines
1313 #define PRQ_RING_MASK ((0x1000 << PRQ_ORDER) - 0x20)
1414 #define PRQ_DEPTH ((0x1000 << PRQ_ORDER) >> 5)
1515
16−/*
17− * The SVM_FLAG_SUPERVISOR_MODE flag requests a PASID which can be used only
18− * for access to kernel addresses. No IOTLB flushes are automatically done
19− * for kernel mappings; it is valid only for access to the kernel's static
20− * 1:1 mapping of physical memory — not to vmalloc or even module mappings.
21− * A future API addition may permit the use of such ranges, by means of an
22− * explicit IOTLB flush call (akin to the DMA API's unmap method).
23− *
24− * It is unlikely that we will ever hook into flush_tlb_kernel_range() to
25− * do such IOTLB flushes automatically.
26− */
27−#define SVM_FLAG_SUPERVISOR_MODE BIT(0)
28−
2916 #endif /* __INTEL_SVM_H__ */
include/linux/interval_tree.h+58−0
@@ -27,4 +27,62 @@ extern struct interval_tree_node * · 26 unmodified lines
2727 interval_tree_iter_next(struct interval_tree_node *node,
2828 unsigned long start, unsigned long last);
2929
30+/**
31+ * struct interval_tree_span_iter - Find used and unused spans.
32+ * @start_hole: Start of an interval for a hole when is_hole == 1
33+ * @last_hole: Inclusive end of an interval for a hole when is_hole == 1
34+ * @start_used: Start of a used interval when is_hole == 0
35+ * @last_used: Inclusive end of a used interval when is_hole == 0
36+ * @is_hole: 0 == used, 1 == is_hole, -1 == done iteration
37+ *
38+ * This iterator travels over spans in an interval tree. It does not return
39+ * nodes but classifies each span as either a hole, where no nodes intersect, or
40+ * a used, which is fully covered by nodes. Each iteration step toggles between
41+ * hole and used until the entire range is covered. The returned spans always
42+ * fully cover the requested range.
43+ *
44+ * The iterator is greedy, it always returns the largest hole or used possible,
45+ * consolidating all consecutive nodes.
46+ *
47+ * Use interval_tree_span_iter_done() to detect end of iteration.
48+ */
49+struct interval_tree_span_iter {
50+ /* private: not for use by the caller */
51+ struct interval_tree_node *nodes[2];
52+ unsigned long first_index;
53+ unsigned long last_index;
54+
55+ /* public: */
56+ union {
57+ unsigned long start_hole;
58+ unsigned long start_used;
59+ };
60+ union {
61+ unsigned long last_hole;
62+ unsigned long last_used;
63+ };
64+ int is_hole;
65+};
66+
67+void interval_tree_span_iter_first(struct interval_tree_span_iter *state,
68+ struct rb_root_cached *itree,
69+ unsigned long first_index,
70+ unsigned long last_index);
71+void interval_tree_span_iter_advance(struct interval_tree_span_iter *iter,
72+ struct rb_root_cached *itree,
73+ unsigned long new_index);
74+void interval_tree_span_iter_next(struct interval_tree_span_iter *state);
75+
76+static inline bool
77+interval_tree_span_iter_done(struct interval_tree_span_iter *state)
78+{
79+ return state->is_hole == -1;
80+}
81+
82+#define interval_tree_for_each_span(span, itree, first_index, last_index) \
83+ for (interval_tree_span_iter_first(span, itree, \
84+ first_index, last_index); \
85+ !interval_tree_span_iter_done(span); \
86+ interval_tree_span_iter_next(span))
87+
3088 #endif /* _LINUX_INTERVAL_TREE_H */
include/linux/iommu.h+114−31
@@ -64,6 +64,8 @@ struct iommu_domain_geometry { · 63 unmodified lines
6464 #define __IOMMU_DOMAIN_PT (1U << 2) /* Domain is identity mapped */
6565 #define __IOMMU_DOMAIN_DMA_FQ (1U << 3) /* DMA-API uses flush queue */
6666
67+#define __IOMMU_DOMAIN_SVA (1U << 4) /* Shared process address space */
68+
6769 /*
6870 * This are the possible domain-types
6971 *
@@ -77,6 +79,8 @@ struct iommu_domain_geometry { · 7 unmodified lines
7779 * certain optimizations for these domains
7880 * IOMMU_DOMAIN_DMA_FQ - As above, but definitely using batched TLB
7981 * invalidation.
82+ * IOMMU_DOMAIN_SVA - DMA addresses are shared process addresses
83+ * represented by mm_struct's.
8084 */
8185 #define IOMMU_DOMAIN_BLOCKED (0U)
8286 #define IOMMU_DOMAIN_IDENTITY (__IOMMU_DOMAIN_PT)
@@ -86,15 +90,27 @@ struct iommu_domain_geometry { · 3 unmodified lines
8690 #define IOMMU_DOMAIN_DMA_FQ (__IOMMU_DOMAIN_PAGING | \
8791 __IOMMU_DOMAIN_DMA_API | \
8892 __IOMMU_DOMAIN_DMA_FQ)
93+#define IOMMU_DOMAIN_SVA (__IOMMU_DOMAIN_SVA)
8994
9095 struct iommu_domain {
9196 unsigned type;
9297 const struct iommu_domain_ops *ops;
9398 unsigned long pgsize_bitmap; /* Bitmap of page sizes in use */
94− iommu_fault_handler_t handler;
95− void *handler_token;
9699 struct iommu_domain_geometry geometry;
97100 struct iommu_dma_cookie *iova_cookie;
101+ enum iommu_page_response_code (*iopf_handler)(struct iommu_fault *fault,
102+ void *data);
103+ void *fault_data;
104+ union {
105+ struct {
106+ iommu_fault_handler_t handler;
107+ void *handler_token;
108+ };
109+ struct { /* IOMMU_DOMAIN_SVA */
110+ struct mm_struct *mm;
111+ int users;
112+ };
113+ };
98114 };
99115
100116 static inline bool iommu_is_dma_domain(struct iommu_domain *domain)
@@ -108,6 +124,11 @@ enum iommu_cap { · 7 unmodified lines
108124 IOMMU_CAP_NOEXEC, /* IOMMU_NOEXEC flag */
109125 IOMMU_CAP_PRE_BOOT_PROTECTION, /* Firmware says it used the IOMMU for
110126 DMA protection and we should too */
127+ /*
128+ * Per-device flag indicating if enforce_cache_coherency() will work on
129+ * this device.
130+ */
131+ IOMMU_CAP_ENFORCE_CACHE_COHERENCY,
111132 };
112133
113134 /* These are the possible reserved region types */
@@ -214,15 +235,15 @@ struct iommu_iotlb_gather { · 100 unmodified lines
214235 * driver init to device driver init (default no)
215236 * @dev_enable/disable_feat: per device entries to enable/disable
216237 * iommu specific features.
217− * @sva_bind: Bind process address space to device
218− * @sva_unbind: Unbind process address space from device
219− * @sva_get_pasid: Get PASID associated to a SVA handle
220238 * @page_response: handle page request response
221239 * @def_domain_type: device default domain type, return value:
222240 * - IOMMU_DOMAIN_IDENTITY: must use an identity domain
223241 * - IOMMU_DOMAIN_DMA: must use a dma domain
224242 * - 0: use the default setting
225243 * @default_domain_ops: the default ops for domains
244+ * @remove_dev_pasid: Remove any translation configurations of a specific
245+ * pasid, so that any DMA transactions with this pasid
246+ * will be blocked by the hardware.
226247 * @pgsize_bitmap: bitmap of all possible supported page sizes
227248 * @owner: Driver module providing these ops
228249 */
@@ -247,16 +268,12 @@ struct iommu_ops { · 18 unmodified lines
247268 int (*dev_enable_feat)(struct device *dev, enum iommu_dev_features f);
248269 int (*dev_disable_feat)(struct device *dev, enum iommu_dev_features f);
249270
250− struct iommu_sva *(*sva_bind)(struct device *dev, struct mm_struct *mm,
251− void *drvdata);
252− void (*sva_unbind)(struct iommu_sva *handle);
253− u32 (*sva_get_pasid)(struct iommu_sva *handle);
254−
255271 int (*page_response)(struct device *dev,
256272 struct iommu_fault_event *evt,
257273 struct iommu_page_response *msg);
258274
259275 int (*def_domain_type)(struct device *dev);
276+ void (*remove_dev_pasid)(struct device *dev, ioasid_t pasid);
260277
261278 const struct iommu_domain_ops *default_domain_ops;
262279 unsigned long pgsize_bitmap;
@@ -266,7 +283,20 @@ struct iommu_ops { · 3 unmodified lines
266283 /**
267284 * struct iommu_domain_ops - domain specific operations
268285 * @attach_dev: attach an iommu domain to a device
286+ * Return:
287+ * * 0 - success
288+ * * EINVAL - can indicate that device and domain are incompatible due to
289+ * some previous configuration of the domain, in which case the
290+ * driver shouldn't log an error, since it is legitimate for a
291+ * caller to test reuse of existing domains. Otherwise, it may
292+ * still represent some other fundamental problem
293+ * * ENOMEM - out of memory
294+ * * ENOSPC - non-ENOMEM type of resource allocation failures
295+ * * EBUSY - device is attached to a domain and cannot be changed
296+ * * ENODEV - device specific errors, not able to be attached
297+ * * <others> - treated as ENODEV by the caller. Use is discouraged
269298 * @detach_dev: detach an iommu domain from a device
299+ * @set_dev_pasid: set an iommu domain to a pasid of device
270300 * @map: map a physically contiguous memory region to an iommu domain
271301 * @map_pages: map a physically contiguous set of pages of the same size to
272302 * an iommu domain.
@@ -287,6 +317,8 @@ struct iommu_ops { · 14 unmodified lines
287317 struct iommu_domain_ops {
288318 int (*attach_dev)(struct iommu_domain *domain, struct device *dev);
289319 void (*detach_dev)(struct iommu_domain *domain, struct device *dev);
320+ int (*set_dev_pasid)(struct iommu_domain *domain, struct device *dev,
321+ ioasid_t pasid);
290322
291323 int (*map)(struct iommu_domain *domain, unsigned long iova,
292324 phys_addr_t paddr, size_t size, int prot, gfp_t gfp);
@@ -322,12 +354,14 @@ struct iommu_domain_ops { · 29 unmodified lines
322354 * @list: Used by the iommu-core to keep a list of registered iommus
323355 * @ops: iommu-ops for talking to this iommu
324356 * @dev: struct device for sysfs handling
357+ * @max_pasids: number of supported PASIDs
325358 */
326359 struct iommu_device {
327360 struct list_head list;
328361 const struct iommu_ops *ops;
329362 struct fwnode_handle *fwnode;
330363 struct device *dev;
364+ u32 max_pasids;
331365 };
332366
333367 /**
@@ -366,6 +400,7 @@ struct iommu_fault_param { · 32 unmodified lines
366400 * @fwspec: IOMMU fwspec data
367401 * @iommu_dev: IOMMU device this device is linked to
368402 * @priv: IOMMU Driver private data
403+ * @max_pasids: number of PASIDs this device can consume
369404 *
370405 * TODO: migrate other per device data pointers under iommu_dev_data, e.g.
371406 * struct iommu_group *iommu_group;
@@ -377,6 +412,7 @@ struct dev_iommu { · 5 unmodified lines
377412 struct iommu_fwspec *fwspec;
378413 struct iommu_device *iommu_dev;
379414 void *priv;
415+ u32 max_pasids;
380416 };
381417
382418 int iommu_device_register(struct iommu_device *iommu,
@@ -626,6 +662,7 @@ struct iommu_fwspec { · 243 unmodified lines
626662 */
627663 struct iommu_sva {
628664 struct device *dev;
665+ struct iommu_domain *domain;
629666 };
630667
631668 int iommu_fwspec_init(struct device *dev, struct fwnode_handle *iommu_fwnode,
@@ -667,12 +704,6 @@ void iommu_release_device(struct device *dev); · 35 unmodified lines
667704 int iommu_dev_enable_feature(struct device *dev, enum iommu_dev_features f);
668705 int iommu_dev_disable_feature(struct device *dev, enum iommu_dev_features f);
669706
670−struct iommu_sva *iommu_sva_bind_device(struct device *dev,
671− struct mm_struct *mm,
672− void *drvdata);
673−void iommu_sva_unbind_device(struct iommu_sva *handle);
674−u32 iommu_sva_get_pasid(struct iommu_sva *handle);
675−
676707 int iommu_device_use_default_domain(struct device *dev);
677708 void iommu_device_unuse_default_domain(struct device *dev);
678709
@@ -680,6 +711,18 @@ int iommu_group_claim_dma_owner(struct iommu_group *group, void *owner); · 1 unmodified line
680711 void iommu_group_release_dma_owner(struct iommu_group *group);
681712 bool iommu_group_dma_owner_claimed(struct iommu_group *group);
682713
714+int iommu_device_claim_dma_owner(struct device *dev, void *owner);
715+void iommu_device_release_dma_owner(struct device *dev);
716+
717+struct iommu_domain *iommu_sva_domain_alloc(struct device *dev,
718+ struct mm_struct *mm);
719+int iommu_attach_device_pasid(struct iommu_domain *domain,
720+ struct device *dev, ioasid_t pasid);
721+void iommu_detach_device_pasid(struct iommu_domain *domain,
722+ struct device *dev, ioasid_t pasid);
723+struct iommu_domain *
724+iommu_get_domain_for_dev_pasid(struct device *dev, ioasid_t pasid,
725+ unsigned int type);
683726 #else /* CONFIG_IOMMU_API */
684727
685728 struct iommu_ops {};
@@ -999,21 +1042,6 @@ iommu_dev_disable_feature(struct device *dev, enum iommu_dev_features feat) · 313 unmodified lines
9991042 return -ENODEV;
10001043 }
10011044
1002−static inline struct iommu_sva *
1003−iommu_sva_bind_device(struct device *dev, struct mm_struct *mm, void *drvdata)
1004−{
1005− return NULL;
1006−}
1007−
1008−static inline void iommu_sva_unbind_device(struct iommu_sva *handle)
1009−{
1010−}
1011−
1012−static inline u32 iommu_sva_get_pasid(struct iommu_sva *handle)
1013−{
1014− return IOMMU_PASID_INVALID;
1015−}
1016−
10171045 static inline struct iommu_fwspec *dev_iommu_fwspec_get(struct device *dev)
10181046 {
10191047 return NULL;
@@ -1042,6 +1070,39 @@ static inline bool iommu_group_dma_owner_claimed(struct iommu_group *group) · 22 unmodified lines
10421070 {
10431071 return false;
10441072 }
1073+
1074+static inline void iommu_device_release_dma_owner(struct device *dev)
1075+{
1076+}
1077+
1078+static inline int iommu_device_claim_dma_owner(struct device *dev, void *owner)
1079+{
1080+ return -ENODEV;
1081+}
1082+
1083+static inline struct iommu_domain *
1084+iommu_sva_domain_alloc(struct device *dev, struct mm_struct *mm)
1085+{
1086+ return NULL;
1087+}
1088+
1089+static inline int iommu_attach_device_pasid(struct iommu_domain *domain,
1090+ struct device *dev, ioasid_t pasid)
1091+{
1092+ return -ENODEV;
1093+}
1094+
1095+static inline void iommu_detach_device_pasid(struct iommu_domain *domain,
1096+ struct device *dev, ioasid_t pasid)
1097+{
1098+}
1099+
1100+static inline struct iommu_domain *
1101+iommu_get_domain_for_dev_pasid(struct device *dev, ioasid_t pasid,
1102+ unsigned int type)
1103+{
1104+ return NULL;
1105+}
10451106 #endif /* CONFIG_IOMMU_API */
10461107
10471108 /**
@@ -1124,4 +1185,26 @@ static inline bool tegra_dev_iommu_get_stream_id(struct device *dev, u32 *stream · 76 unmodified lines
11241185 return false;
11251186 }
11261187
1188+#ifdef CONFIG_IOMMU_SVA
1189+struct iommu_sva *iommu_sva_bind_device(struct device *dev,
1190+ struct mm_struct *mm);
1191+void iommu_sva_unbind_device(struct iommu_sva *handle);
1192+u32 iommu_sva_get_pasid(struct iommu_sva *handle);
1193+#else
1194+static inline struct iommu_sva *
1195+iommu_sva_bind_device(struct device *dev, struct mm_struct *mm)
1196+{
1197+ return NULL;
1198+}
1199+
1200+static inline void iommu_sva_unbind_device(struct iommu_sva *handle)
1201+{
1202+}
1203+
1204+static inline u32 iommu_sva_get_pasid(struct iommu_sva *handle)
1205+{
1206+ return IOMMU_PASID_INVALID;
1207+}
1208+#endif /* CONFIG_IOMMU_SVA */
1209+
11271210 #endif /* __LINUX_IOMMU_H */
include/linux/iommufd.h+98−0
@@ -0,0 +1,98 @@
1+/* SPDX-License-Identifier: GPL-2.0-only */
2+/*
3+ * Copyright (C) 2021 Intel Corporation
4+ * Copyright (c) 2021-2022, NVIDIA CORPORATION & AFFILIATES
5+ */
6+#ifndef __LINUX_IOMMUFD_H
7+#define __LINUX_IOMMUFD_H
8+
9+#include <linux/types.h>
10+#include <linux/errno.h>
11+#include <linux/err.h>
12+
13+struct device;
14+struct iommufd_device;
15+struct page;
16+struct iommufd_ctx;
17+struct iommufd_access;
18+struct file;
19+
20+struct iommufd_device *iommufd_device_bind(struct iommufd_ctx *ictx,
21+ struct device *dev, u32 *id);
22+void iommufd_device_unbind(struct iommufd_device *idev);
23+
24+int iommufd_device_attach(struct iommufd_device *idev, u32 *pt_id);
25+void iommufd_device_detach(struct iommufd_device *idev);
26+
27+struct iommufd_access_ops {
28+ u8 needs_pin_pages : 1;
29+ void (*unmap)(void *data, unsigned long iova, unsigned long length);
30+};
31+
32+enum {
33+ IOMMUFD_ACCESS_RW_READ = 0,
34+ IOMMUFD_ACCESS_RW_WRITE = 1 << 0,
35+ /* Set if the caller is in a kthread then rw will use kthread_use_mm() */
36+ IOMMUFD_ACCESS_RW_KTHREAD = 1 << 1,
37+
38+ /* Only for use by selftest */
39+ __IOMMUFD_ACCESS_RW_SLOW_PATH = 1 << 2,
40+};
41+
42+struct iommufd_access *
43+iommufd_access_create(struct iommufd_ctx *ictx, u32 ioas_id,
44+ const struct iommufd_access_ops *ops, void *data);
45+void iommufd_access_destroy(struct iommufd_access *access);
46+
47+void iommufd_ctx_get(struct iommufd_ctx *ictx);
48+
49+#if IS_ENABLED(CONFIG_IOMMUFD)
50+struct iommufd_ctx *iommufd_ctx_from_file(struct file *file);
51+void iommufd_ctx_put(struct iommufd_ctx *ictx);
52+
53+int iommufd_access_pin_pages(struct iommufd_access *access, unsigned long iova,
54+ unsigned long length, struct page **out_pages,
55+ unsigned int flags);
56+void iommufd_access_unpin_pages(struct iommufd_access *access,
57+ unsigned long iova, unsigned long length);
58+int iommufd_access_rw(struct iommufd_access *access, unsigned long iova,
59+ void *data, size_t len, unsigned int flags);
60+int iommufd_vfio_compat_ioas_id(struct iommufd_ctx *ictx, u32 *out_ioas_id);
61+#else /* !CONFIG_IOMMUFD */
62+static inline struct iommufd_ctx *iommufd_ctx_from_file(struct file *file)
63+{
64+ return ERR_PTR(-EOPNOTSUPP);
65+}
66+
67+static inline void iommufd_ctx_put(struct iommufd_ctx *ictx)
68+{
69+}
70+
71+static inline int iommufd_access_pin_pages(struct iommufd_access *access,
72+ unsigned long iova,
73+ unsigned long length,
74+ struct page **out_pages,
75+ unsigned int flags)
76+{
77+ return -EOPNOTSUPP;
78+}
79+
80+static inline void iommufd_access_unpin_pages(struct iommufd_access *access,
81+ unsigned long iova,
82+ unsigned long length)
83+{
84+}
85+
86+static inline int iommufd_access_rw(struct iommufd_access *access, unsigned long iova,
87+ void *data, size_t len, unsigned int flags)
88+{
89+ return -EOPNOTSUPP;
90+}
91+
92+static inline int iommufd_vfio_compat_ioas_id(struct iommufd_ctx *ictx,
93+ u32 *out_ioas_id)
94+{
95+ return -EOPNOTSUPP;
96+}
97+#endif /* CONFIG_IOMMUFD */
98+#endif
include/linux/sched/user.h+1−1
@@ -25,7 +25,7 @@ struct user_struct { · 24 unmodified lines
2525
2626 #if defined(CONFIG_PERF_EVENTS) || defined(CONFIG_BPF_SYSCALL) || \
2727 defined(CONFIG_NET) || defined(CONFIG_IO_URING) || \
28− defined(CONFIG_VFIO_PCI_ZDEV_KVM)
28+ defined(CONFIG_VFIO_PCI_ZDEV_KVM) || IS_ENABLED(CONFIG_IOMMUFD)
2929 atomic_long_t locked_vm;
3030 #endif
3131 #ifdef CONFIG_WATCH_QUEUE
include/linux/vfio.h+39−0
@@ -17,6 +17,9 @@ · 16 unmodified lines
1717 #include <linux/iova_bitmap.h>
1818
1919 struct kvm;
20+struct iommufd_ctx;
21+struct iommufd_device;
22+struct iommufd_access;
2023
2124 /*
2225 * VFIO devices can be placed in a set, this allows all devices to share this
@@ -54,6 +57,12 @@ struct vfio_device { · 31 unmodified lines
5457 struct completion comp;
5558 struct list_head group_next;
5659 struct list_head iommu_entry;
60+ struct iommufd_access *iommufd_access;
61+#if IS_ENABLED(CONFIG_IOMMUFD)
62+ struct iommufd_device *iommufd_device;
63+ struct iommufd_ctx *iommufd_ictx;
64+ bool iommufd_attached;
65+#endif
5766 };
5867
5968 /**
@@ -80,6 +89,10 @@ struct vfio_device_ops { · 20 unmodified lines
8089 char *name;
8190 int (*init)(struct vfio_device *vdev);
8291 void (*release)(struct vfio_device *vdev);
92+ int (*bind_iommufd)(struct vfio_device *vdev,
93+ struct iommufd_ctx *ictx, u32 *out_device_id);
94+ void (*unbind_iommufd)(struct vfio_device *vdev);
95+ int (*attach_ioas)(struct vfio_device *vdev, u32 *pt_id);
8396 int (*open_device)(struct vfio_device *vdev);
8497 void (*close_device)(struct vfio_device *vdev);
8598 ssize_t (*read)(struct vfio_device *vdev, char __user *buf,
@@ -96,6 +109,32 @@ struct vfio_device_ops { · 10 unmodified lines
96109 void __user *arg, size_t argsz);
97110 };
98111
112+#if IS_ENABLED(CONFIG_IOMMUFD)
113+int vfio_iommufd_physical_bind(struct vfio_device *vdev,
114+ struct iommufd_ctx *ictx, u32 *out_device_id);
115+void vfio_iommufd_physical_unbind(struct vfio_device *vdev);
116+int vfio_iommufd_physical_attach_ioas(struct vfio_device *vdev, u32 *pt_id);
117+int vfio_iommufd_emulated_bind(struct vfio_device *vdev,
118+ struct iommufd_ctx *ictx, u32 *out_device_id);
119+void vfio_iommufd_emulated_unbind(struct vfio_device *vdev);
120+int vfio_iommufd_emulated_attach_ioas(struct vfio_device *vdev, u32 *pt_id);
121+#else
122+#define vfio_iommufd_physical_bind \
123+ ((int (*)(struct vfio_device *vdev, struct iommufd_ctx *ictx, \
124+ u32 *out_device_id)) NULL)
125+#define vfio_iommufd_physical_unbind \
126+ ((void (*)(struct vfio_device *vdev)) NULL)
127+#define vfio_iommufd_physical_attach_ioas \
128+ ((int (*)(struct vfio_device *vdev, u32 *pt_id)) NULL)
129+#define vfio_iommufd_emulated_bind \
130+ ((int (*)(struct vfio_device *vdev, struct iommufd_ctx *ictx, \
131+ u32 *out_device_id)) NULL)
132+#define vfio_iommufd_emulated_unbind \
133+ ((void (*)(struct vfio_device *vdev)) NULL)
134+#define vfio_iommufd_emulated_attach_ioas \
135+ ((int (*)(struct vfio_device *vdev, u32 *pt_id)) NULL)
136+#endif
137+
99138 /**
100139 * @migration_set_state: Optional callback to change the migration state for
101140 * devices that support migration. It's mandatory for
include/uapi/linux/iommufd.h+347−0
@@ -0,0 +1,347 @@
1+/* SPDX-License-Identifier: GPL-2.0 WITH Linux-syscall-note */
2+/* Copyright (c) 2021-2022, NVIDIA CORPORATION & AFFILIATES.
3+ */
4+#ifndef _UAPI_IOMMUFD_H
5+#define _UAPI_IOMMUFD_H
6+
7+#include <linux/types.h>
8+#include <linux/ioctl.h>
9+
10+#define IOMMUFD_TYPE (';')
11+
12+/**
13+ * DOC: General ioctl format
14+ *
15+ * The ioctl interface follows a general format to allow for extensibility. Each
16+ * ioctl is passed in a structure pointer as the argument providing the size of
17+ * the structure in the first u32. The kernel checks that any structure space
18+ * beyond what it understands is 0. This allows userspace to use the backward
19+ * compatible portion while consistently using the newer, larger, structures.
20+ *
21+ * ioctls use a standard meaning for common errnos:
22+ *
23+ * - ENOTTY: The IOCTL number itself is not supported at all
24+ * - E2BIG: The IOCTL number is supported, but the provided structure has
25+ * non-zero in a part the kernel does not understand.
26+ * - EOPNOTSUPP: The IOCTL number is supported, and the structure is
27+ * understood, however a known field has a value the kernel does not
28+ * understand or support.
29+ * - EINVAL: Everything about the IOCTL was understood, but a field is not
30+ * correct.
31+ * - ENOENT: An ID or IOVA provided does not exist.
32+ * - ENOMEM: Out of memory.
33+ * - EOVERFLOW: Mathematics overflowed.
34+ *
35+ * As well as additional errnos, within specific ioctls.
36+ */
37+enum {
38+ IOMMUFD_CMD_BASE = 0x80,
39+ IOMMUFD_CMD_DESTROY = IOMMUFD_CMD_BASE,
40+ IOMMUFD_CMD_IOAS_ALLOC,
41+ IOMMUFD_CMD_IOAS_ALLOW_IOVAS,
42+ IOMMUFD_CMD_IOAS_COPY,
43+ IOMMUFD_CMD_IOAS_IOVA_RANGES,
44+ IOMMUFD_CMD_IOAS_MAP,
45+ IOMMUFD_CMD_IOAS_UNMAP,
46+ IOMMUFD_CMD_OPTION,
47+ IOMMUFD_CMD_VFIO_IOAS,
48+};
49+
50+/**
51+ * struct iommu_destroy - ioctl(IOMMU_DESTROY)
52+ * @size: sizeof(struct iommu_destroy)
53+ * @id: iommufd object ID to destroy. Can be any destroyable object type.
54+ *
55+ * Destroy any object held within iommufd.
56+ */
57+struct iommu_destroy {
58+ __u32 size;
59+ __u32 id;
60+};
61+#define IOMMU_DESTROY _IO(IOMMUFD_TYPE, IOMMUFD_CMD_DESTROY)
62+
63+/**
64+ * struct iommu_ioas_alloc - ioctl(IOMMU_IOAS_ALLOC)
65+ * @size: sizeof(struct iommu_ioas_alloc)
66+ * @flags: Must be 0
67+ * @out_ioas_id: Output IOAS ID for the allocated object
68+ *
69+ * Allocate an IO Address Space (IOAS) which holds an IO Virtual Address (IOVA)
70+ * to memory mapping.
71+ */
72+struct iommu_ioas_alloc {
73+ __u32 size;
74+ __u32 flags;
75+ __u32 out_ioas_id;
76+};
77+#define IOMMU_IOAS_ALLOC _IO(IOMMUFD_TYPE, IOMMUFD_CMD_IOAS_ALLOC)
78+
79+/**
80+ * struct iommu_iova_range - ioctl(IOMMU_IOVA_RANGE)
81+ * @start: First IOVA
82+ * @last: Inclusive last IOVA
83+ *
84+ * An interval in IOVA space.
85+ */
86+struct iommu_iova_range {
87+ __aligned_u64 start;
88+ __aligned_u64 last;
89+};
90+
91+/**
92+ * struct iommu_ioas_iova_ranges - ioctl(IOMMU_IOAS_IOVA_RANGES)
93+ * @size: sizeof(struct iommu_ioas_iova_ranges)
94+ * @ioas_id: IOAS ID to read ranges from
95+ * @num_iovas: Input/Output total number of ranges in the IOAS
96+ * @__reserved: Must be 0
97+ * @allowed_iovas: Pointer to the output array of struct iommu_iova_range
98+ * @out_iova_alignment: Minimum alignment required for mapping IOVA
99+ *
100+ * Query an IOAS for ranges of allowed IOVAs. Mapping IOVA outside these ranges
101+ * is not allowed. num_iovas will be set to the total number of iovas and
102+ * the allowed_iovas[] will be filled in as space permits.
103+ *
104+ * The allowed ranges are dependent on the HW path the DMA operation takes, and
105+ * can change during the lifetime of the IOAS. A fresh empty IOAS will have a
106+ * full range, and each attached device will narrow the ranges based on that
107+ * device's HW restrictions. Detaching a device can widen the ranges. Userspace
108+ * should query ranges after every attach/detach to know what IOVAs are valid
109+ * for mapping.
110+ *
111+ * On input num_iovas is the length of the allowed_iovas array. On output it is
112+ * the total number of iovas filled in. The ioctl will return -EMSGSIZE and set
113+ * num_iovas to the required value if num_iovas is too small. In this case the
114+ * caller should allocate a larger output array and re-issue the ioctl.
115+ *
116+ * out_iova_alignment returns the minimum IOVA alignment that can be given
117+ * to IOMMU_IOAS_MAP/COPY. IOVA's must satisfy::
118+ *
119+ * starting_iova % out_iova_alignment == 0
120+ * (starting_iova + length) % out_iova_alignment == 0
121+ *
122+ * out_iova_alignment can be 1 indicating any IOVA is allowed. It cannot
123+ * be higher than the system PAGE_SIZE.
124+ */
125+struct iommu_ioas_iova_ranges {
126+ __u32 size;
127+ __u32 ioas_id;
128+ __u32 num_iovas;
129+ __u32 __reserved;
130+ __aligned_u64 allowed_iovas;
131+ __aligned_u64 out_iova_alignment;
132+};
133+#define IOMMU_IOAS_IOVA_RANGES _IO(IOMMUFD_TYPE, IOMMUFD_CMD_IOAS_IOVA_RANGES)
134+
135+/**
136+ * struct iommu_ioas_allow_iovas - ioctl(IOMMU_IOAS_ALLOW_IOVAS)
137+ * @size: sizeof(struct iommu_ioas_allow_iovas)
138+ * @ioas_id: IOAS ID to allow IOVAs from
139+ * @num_iovas: Input/Output total number of ranges in the IOAS
140+ * @__reserved: Must be 0
141+ * @allowed_iovas: Pointer to array of struct iommu_iova_range
142+ *
143+ * Ensure a range of IOVAs are always available for allocation. If this call
144+ * succeeds then IOMMU_IOAS_IOVA_RANGES will never return a list of IOVA ranges
145+ * that are narrower than the ranges provided here. This call will fail if
146+ * IOMMU_IOAS_IOVA_RANGES is currently narrower than the given ranges.
147+ *
148+ * When an IOAS is first created the IOVA_RANGES will be maximally sized, and as
149+ * devices are attached the IOVA will narrow based on the device restrictions.
150+ * When an allowed range is specified any narrowing will be refused, ie device
151+ * attachment can fail if the device requires limiting within the allowed range.
152+ *
153+ * Automatic IOVA allocation is also impacted by this call. MAP will only
154+ * allocate within the allowed IOVAs if they are present.
155+ *
156+ * This call replaces the entire allowed list with the given list.
157+ */
158+struct iommu_ioas_allow_iovas {
159+ __u32 size;
160+ __u32 ioas_id;
161+ __u32 num_iovas;
162+ __u32 __reserved;
163+ __aligned_u64 allowed_iovas;
164+};
165+#define IOMMU_IOAS_ALLOW_IOVAS _IO(IOMMUFD_TYPE, IOMMUFD_CMD_IOAS_ALLOW_IOVAS)
166+
167+/**
168+ * enum iommufd_ioas_map_flags - Flags for map and copy
169+ * @IOMMU_IOAS_MAP_FIXED_IOVA: If clear the kernel will compute an appropriate
170+ * IOVA to place the mapping at
171+ * @IOMMU_IOAS_MAP_WRITEABLE: DMA is allowed to write to this mapping
172+ * @IOMMU_IOAS_MAP_READABLE: DMA is allowed to read from this mapping
173+ */
174+enum iommufd_ioas_map_flags {
175+ IOMMU_IOAS_MAP_FIXED_IOVA = 1 << 0,
176+ IOMMU_IOAS_MAP_WRITEABLE = 1 << 1,
177+ IOMMU_IOAS_MAP_READABLE = 1 << 2,
178+};
179+
180+/**
181+ * struct iommu_ioas_map - ioctl(IOMMU_IOAS_MAP)
182+ * @size: sizeof(struct iommu_ioas_map)
183+ * @flags: Combination of enum iommufd_ioas_map_flags
184+ * @ioas_id: IOAS ID to change the mapping of
185+ * @__reserved: Must be 0
186+ * @user_va: Userspace pointer to start mapping from
187+ * @length: Number of bytes to map
188+ * @iova: IOVA the mapping was placed at. If IOMMU_IOAS_MAP_FIXED_IOVA is set
189+ * then this must be provided as input.
190+ *
191+ * Set an IOVA mapping from a user pointer. If FIXED_IOVA is specified then the
192+ * mapping will be established at iova, otherwise a suitable location based on
193+ * the reserved and allowed lists will be automatically selected and returned in
194+ * iova.
195+ *
196+ * If IOMMU_IOAS_MAP_FIXED_IOVA is specified then the iova range must currently
197+ * be unused, existing IOVA cannot be replaced.
198+ */
199+struct iommu_ioas_map {
200+ __u32 size;
201+ __u32 flags;
202+ __u32 ioas_id;
203+ __u32 __reserved;
204+ __aligned_u64 user_va;
205+ __aligned_u64 length;
206+ __aligned_u64 iova;
207+};
208+#define IOMMU_IOAS_MAP _IO(IOMMUFD_TYPE, IOMMUFD_CMD_IOAS_MAP)
209+
210+/**
211+ * struct iommu_ioas_copy - ioctl(IOMMU_IOAS_COPY)
212+ * @size: sizeof(struct iommu_ioas_copy)
213+ * @flags: Combination of enum iommufd_ioas_map_flags
214+ * @dst_ioas_id: IOAS ID to change the mapping of
215+ * @src_ioas_id: IOAS ID to copy from
216+ * @length: Number of bytes to copy and map
217+ * @dst_iova: IOVA the mapping was placed at. If IOMMU_IOAS_MAP_FIXED_IOVA is
218+ * set then this must be provided as input.
219+ * @src_iova: IOVA to start the copy
220+ *
221+ * Copy an already existing mapping from src_ioas_id and establish it in
222+ * dst_ioas_id. The src iova/length must exactly match a range used with
223+ * IOMMU_IOAS_MAP.
224+ *
225+ * This may be used to efficiently clone a subset of an IOAS to another, or as a
226+ * kind of 'cache' to speed up mapping. Copy has an efficiency advantage over
227+ * establishing equivalent new mappings, as internal resources are shared, and
228+ * the kernel will pin the user memory only once.
229+ */
230+struct iommu_ioas_copy {
231+ __u32 size;
232+ __u32 flags;
233+ __u32 dst_ioas_id;
234+ __u32 src_ioas_id;
235+ __aligned_u64 length;
236+ __aligned_u64 dst_iova;
237+ __aligned_u64 src_iova;
238+};
239+#define IOMMU_IOAS_COPY _IO(IOMMUFD_TYPE, IOMMUFD_CMD_IOAS_COPY)
240+
241+/**
242+ * struct iommu_ioas_unmap - ioctl(IOMMU_IOAS_UNMAP)
243+ * @size: sizeof(struct iommu_ioas_unmap)
244+ * @ioas_id: IOAS ID to change the mapping of
245+ * @iova: IOVA to start the unmapping at
246+ * @length: Number of bytes to unmap, and return back the bytes unmapped
247+ *
248+ * Unmap an IOVA range. The iova/length must be a superset of a previously
249+ * mapped range used with IOMMU_IOAS_MAP or IOMMU_IOAS_COPY. Splitting or
250+ * truncating ranges is not allowed. The values 0 to U64_MAX will unmap
251+ * everything.
252+ */
253+struct iommu_ioas_unmap {
254+ __u32 size;
255+ __u32 ioas_id;
256+ __aligned_u64 iova;
257+ __aligned_u64 length;
258+};
259+#define IOMMU_IOAS_UNMAP _IO(IOMMUFD_TYPE, IOMMUFD_CMD_IOAS_UNMAP)
260+
261+/**
262+ * enum iommufd_option - ioctl(IOMMU_OPTION_RLIMIT_MODE) and
263+ * ioctl(IOMMU_OPTION_HUGE_PAGES)
264+ * @IOMMU_OPTION_RLIMIT_MODE:
265+ * Change how RLIMIT_MEMLOCK accounting works. The caller must have privilege
266+ * to invoke this. Value 0 (default) is user based accouting, 1 uses process
267+ * based accounting. Global option, object_id must be 0
268+ * @IOMMU_OPTION_HUGE_PAGES:
269+ * Value 1 (default) allows contiguous pages to be combined when generating
270+ * iommu mappings. Value 0 disables combining, everything is mapped to
271+ * PAGE_SIZE. This can be useful for benchmarking. This is a per-IOAS
272+ * option, the object_id must be the IOAS ID.
273+ */
274+enum iommufd_option {
275+ IOMMU_OPTION_RLIMIT_MODE = 0,
276+ IOMMU_OPTION_HUGE_PAGES = 1,
277+};
278+
279+/**
280+ * enum iommufd_option_ops - ioctl(IOMMU_OPTION_OP_SET) and
281+ * ioctl(IOMMU_OPTION_OP_GET)
282+ * @IOMMU_OPTION_OP_SET: Set the option's value
283+ * @IOMMU_OPTION_OP_GET: Get the option's value
284+ */
285+enum iommufd_option_ops {
286+ IOMMU_OPTION_OP_SET = 0,
287+ IOMMU_OPTION_OP_GET = 1,
288+};
289+
290+/**
291+ * struct iommu_option - iommu option multiplexer
292+ * @size: sizeof(struct iommu_option)
293+ * @option_id: One of enum iommufd_option
294+ * @op: One of enum iommufd_option_ops
295+ * @__reserved: Must be 0
296+ * @object_id: ID of the object if required
297+ * @val64: Option value to set or value returned on get
298+ *
299+ * Change a simple option value. This multiplexor allows controlling options
300+ * on objects. IOMMU_OPTION_OP_SET will load an option and IOMMU_OPTION_OP_GET
301+ * will return the current value.
302+ */
303+struct iommu_option {
304+ __u32 size;
305+ __u32 option_id;
306+ __u16 op;
307+ __u16 __reserved;
308+ __u32 object_id;
309+ __aligned_u64 val64;
310+};
311+#define IOMMU_OPTION _IO(IOMMUFD_TYPE, IOMMUFD_CMD_OPTION)
312+
313+/**
314+ * enum iommufd_vfio_ioas_op - IOMMU_VFIO_IOAS_* ioctls
315+ * @IOMMU_VFIO_IOAS_GET: Get the current compatibility IOAS
316+ * @IOMMU_VFIO_IOAS_SET: Change the current compatibility IOAS
317+ * @IOMMU_VFIO_IOAS_CLEAR: Disable VFIO compatibility
318+ */
319+enum iommufd_vfio_ioas_op {
320+ IOMMU_VFIO_IOAS_GET = 0,
321+ IOMMU_VFIO_IOAS_SET = 1,
322+ IOMMU_VFIO_IOAS_CLEAR = 2,
323+};
324+
325+/**
326+ * struct iommu_vfio_ioas - ioctl(IOMMU_VFIO_IOAS)
327+ * @size: sizeof(struct iommu_vfio_ioas)
328+ * @ioas_id: For IOMMU_VFIO_IOAS_SET the input IOAS ID to set
329+ * For IOMMU_VFIO_IOAS_GET will output the IOAS ID
330+ * @op: One of enum iommufd_vfio_ioas_op
331+ * @__reserved: Must be 0
332+ *
333+ * The VFIO compatibility support uses a single ioas because VFIO APIs do not
334+ * support the ID field. Set or Get the IOAS that VFIO compatibility will use.
335+ * When VFIO_GROUP_SET_CONTAINER is used on an iommufd it will get the
336+ * compatibility ioas, either by taking what is already set, or auto creating
337+ * one. From then on VFIO will continue to use that ioas and is not effected by
338+ * this ioctl. SET or CLEAR does not destroy any auto-created IOAS.
339+ */
340+struct iommu_vfio_ioas {
341+ __u32 size;
342+ __u32 ioas_id;
343+ __u16 op;
344+ __u16 __reserved;
345+};
346+#define IOMMU_VFIO_IOAS _IO(IOMMUFD_TYPE, IOMMUFD_CMD_VFIO_IOAS)
347+#endif
kernel/user.c+1−0
@@ -185,6 +185,7 @@ void free_uid(struct user_struct *up) · 184 unmodified lines
185185 if (refcount_dec_and_lock_irqsave(&up->__count, &uidhash_lock, &flags))
186186 free_user(up, flags);
187187 }
188+EXPORT_SYMBOL_GPL(free_uid);
188189
189190 struct user_struct *alloc_uid(kuid_t uid)
190191 {
lib/Kconfig+4−0
@@ -480,6 +480,10 @@ config INTERVAL_TREE · 479 unmodified lines
480480
481481 for more information.
482482
483+config INTERVAL_TREE_SPAN_ITER
484+ bool
485+ depends on INTERVAL_TREE
486+
483487 config XARRAY_MULTI
484488 bool
485489 help
lib/interval_tree.c+132−0
@@ -15,3 +15,135 @@ EXPORT_SYMBOL_GPL(interval_tree_insert); · 14 unmodified lines
1515 EXPORT_SYMBOL_GPL(interval_tree_remove);
1616 EXPORT_SYMBOL_GPL(interval_tree_iter_first);
1717 EXPORT_SYMBOL_GPL(interval_tree_iter_next);
18+
19+#ifdef CONFIG_INTERVAL_TREE_SPAN_ITER
20+/*
21+ * Roll nodes[1] into nodes[0] by advancing nodes[1] to the end of a contiguous
22+ * span of nodes. This makes nodes[0]->last the end of that contiguous used span
23+ * indexes that started at the original nodes[1]->start. nodes[1] is now the
24+ * first node starting the next used span. A hole span is between nodes[0]->last
25+ * and nodes[1]->start. nodes[1] must be !NULL.
26+ */
27+static void
28+interval_tree_span_iter_next_gap(struct interval_tree_span_iter *state)
29+{
30+ struct interval_tree_node *cur = state->nodes[1];
31+
32+ state->nodes[0] = cur;
33+ do {
34+ if (cur->last > state->nodes[0]->last)
35+ state->nodes[0] = cur;
36+ cur = interval_tree_iter_next(cur, state->first_index,
37+ state->last_index);
38+ } while (cur && (state->nodes[0]->last >= cur->start ||
39+ state->nodes[0]->last + 1 == cur->start));
40+ state->nodes[1] = cur;
41+}
42+
43+void interval_tree_span_iter_first(struct interval_tree_span_iter *iter,
44+ struct rb_root_cached *itree,
45+ unsigned long first_index,
46+ unsigned long last_index)
47+{
48+ iter->first_index = first_index;
49+ iter->last_index = last_index;
50+ iter->nodes[0] = NULL;
51+ iter->nodes[1] =
52+ interval_tree_iter_first(itree, first_index, last_index);
53+ if (!iter->nodes[1]) {
54+ /* No nodes intersect the span, whole span is hole */
55+ iter->start_hole = first_index;
56+ iter->last_hole = last_index;
57+ iter->is_hole = 1;
58+ return;
59+ }
60+ if (iter->nodes[1]->start > first_index) {
61+ /* Leading hole on first iteration */
62+ iter->start_hole = first_index;
63+ iter->last_hole = iter->nodes[1]->start - 1;
64+ iter->is_hole = 1;
65+ interval_tree_span_iter_next_gap(iter);
66+ return;
67+ }
68+
69+ /* Starting inside a used */
70+ iter->start_used = first_index;
71+ iter->is_hole = 0;
72+ interval_tree_span_iter_next_gap(iter);
73+ iter->last_used = iter->nodes[0]->last;
74+ if (iter->last_used >= last_index) {
75+ iter->last_used = last_index;
76+ iter->nodes[0] = NULL;
77+ iter->nodes[1] = NULL;
78+ }
79+}
80+EXPORT_SYMBOL_GPL(interval_tree_span_iter_first);
81+
82+void interval_tree_span_iter_next(struct interval_tree_span_iter *iter)
83+{
84+ if (!iter->nodes[0] && !iter->nodes[1]) {
85+ iter->is_hole = -1;
86+ return;
87+ }
88+
89+ if (iter->is_hole) {
90+ iter->start_used = iter->last_hole + 1;
91+ iter->last_used = iter->nodes[0]->last;
92+ if (iter->last_used >= iter->last_index) {
93+ iter->last_used = iter->last_index;
94+ iter->nodes[0] = NULL;
95+ iter->nodes[1] = NULL;
96+ }
97+ iter->is_hole = 0;
98+ return;
99+ }
100+
101+ if (!iter->nodes[1]) {
102+ /* Trailing hole */
103+ iter->start_hole = iter->nodes[0]->last + 1;
104+ iter->last_hole = iter->last_index;
105+ iter->nodes[0] = NULL;
106+ iter->is_hole = 1;
107+ return;
108+ }
109+
110+ /* must have both nodes[0] and [1], interior hole */
111+ iter->start_hole = iter->nodes[0]->last + 1;
112+ iter->last_hole = iter->nodes[1]->start - 1;
113+ iter->is_hole = 1;
114+ interval_tree_span_iter_next_gap(iter);
115+}
116+EXPORT_SYMBOL_GPL(interval_tree_span_iter_next);
117+
118+/*
119+ * Advance the iterator index to a specific position. The returned used/hole is
120+ * updated to start at new_index. This is faster than calling
121+ * interval_tree_span_iter_first() as it can avoid full searches in several
122+ * cases where the iterator is already set.
123+ */
124+void interval_tree_span_iter_advance(struct interval_tree_span_iter *iter,
125+ struct rb_root_cached *itree,
126+ unsigned long new_index)
127+{
128+ if (iter->is_hole == -1)
129+ return;
130+
131+ iter->first_index = new_index;
132+ if (new_index > iter->last_index) {
133+ iter->is_hole = -1;
134+ return;
135+ }
136+
137+ /* Rely on the union aliasing hole/used */
138+ if (iter->start_hole <= new_index && new_index <= iter->last_hole) {
139+ iter->start_hole = new_index;
140+ return;
141+ }
142+ if (new_index == iter->last_hole + 1)
143+ interval_tree_span_iter_next(iter);
144+ else
145+ interval_tree_span_iter_first(iter, itree, new_index,
146+ iter->last_index);
147+}
148+EXPORT_SYMBOL_GPL(interval_tree_span_iter_advance);
149+#endif
scripts/kernel-doc+8−4
@@ -256,6 +256,7 @@ my $doc_inline_sect = '\s*\*\s*(@\s*[\w][\w\.]*\s*):(.*)'; · 255 unmodified lines
256256 my $doc_inline_end = '^\s*\*/\s*$';
257257 my $doc_inline_oneline = '^\s*/\*\*\s*(@[\w\s]+):\s*(.*)\s*\*/\s*$';
258258 my $export_symbol = '^\s*EXPORT_SYMBOL(_GPL)?\s*\(\s*(\w+)\s*\)\s*;';
259+my $export_symbol_ns = '^\s*EXPORT_SYMBOL_NS(_GPL)?\s*\(\s*(\w+)\s*,\s*\w+\)\s*;';
259260 my $function_pointer = qr{([^\(]*\(\*)\s*\)\s*\(([^\)]*)\)};
260261 my $attribute = qr{__attribute__\s*\(\([a-z0-9,_\*\s\(\)]*\)\)}i;
261262
@@ -1960,6 +1961,10 @@ sub process_export_file($) { · 1698 unmodified lines
19601961 next if (defined($nosymbol_table{$2}));
19611962 $function_table{$2} = 1;
19621963 }
1964+ if (/$export_symbol_ns/) {
1965+ next if (defined($nosymbol_table{$2}));
1966+ $function_table{$2} = 1;
1967+ }
19631968 }
19641969
19651970 close(IN);
@@ -2431,12 +2436,12 @@ found on PATH. · 465 unmodified lines
24312436 =item -export
24322437
24332438 Only output documentation for the symbols that have been exported using
2434−EXPORT_SYMBOL() or EXPORT_SYMBOL_GPL() in any input FILE or -export-file FILE.
2439+EXPORT_SYMBOL() and related macros in any input FILE or -export-file FILE.
24352440
24362441 =item -internal
24372442
24382443 Only output documentation for the symbols that have NOT been exported using
2439−EXPORT_SYMBOL() or EXPORT_SYMBOL_GPL() in any input FILE or -export-file FILE.
2444+EXPORT_SYMBOL() and related macros in any input FILE or -export-file FILE.
24402445
24412446 =item -function NAME
24422447
@@ -2463,8 +2468,7 @@ Do not output DOC: sections. · 20 unmodified lines
24632468
24642469 =item -export-file FILE
24652470
2466−Specify an additional FILE in which to look for EXPORT_SYMBOL() and
2467−EXPORT_SYMBOL_GPL().
2471+Specify an additional FILE in which to look for EXPORT_SYMBOL information.
24682472
24692473 To be used with -export or -internal.
24702474
tools/testing/selftests/Makefile+1−0
@@ -27,6 +27,7 @@ TARGETS += ftrace · 26 unmodified lines
2727 TARGETS += futex
2828 TARGETS += gpio
2929 TARGETS += intel_pstate
30+TARGETS += iommu
3031 TARGETS += ipc
3132 TARGETS += ir
3233 TARGETS += kcmp
tools/testing/selftests/iommu/.gitignore+3−0
@@ -0,0 +1,3 @@
1+# SPDX-License-Identifier: GPL-2.0-only
2+/iommufd
3+/iommufd_fail_nth
tools/testing/selftests/iommu/Makefile+12−0
@@ -0,0 +1,12 @@
1+# SPDX-License-Identifier: GPL-2.0-only
2+CFLAGS += -Wall -O2 -Wno-unused-function
3+CFLAGS += -I../../../../include/uapi/
4+CFLAGS += -I../../../../include/
5+
6+CFLAGS += -D_GNU_SOURCE
7+
8+TEST_GEN_PROGS :=
9+TEST_GEN_PROGS += iommufd
10+TEST_GEN_PROGS += iommufd_fail_nth
11+
12+include ../lib.mk
tools/testing/selftests/iommu/config+2−0
@@ -0,0 +1,2 @@
1+CONFIG_IOMMUFD
2+CONFIG_IOMMUFD_TEST
tools/testing/selftests/iommu/iommufd.c+1654−0
@@ -0,0 +1,1654 @@
1+// SPDX-License-Identifier: GPL-2.0-only
2+/* Copyright (c) 2021-2022, NVIDIA CORPORATION & AFFILIATES */
3+#include <stdlib.h>
4+#include <sys/mman.h>
5+#include <sys/eventfd.h>
6+
7+#define __EXPORTED_HEADERS__
8+#include <linux/vfio.h>
9+
10+#include "iommufd_utils.h"
11+
12+static void *buffer;
13+
14+static unsigned long PAGE_SIZE;
15+static unsigned long HUGEPAGE_SIZE;
16+
17+#define MOCK_PAGE_SIZE (PAGE_SIZE / 2)
18+
19+static unsigned long get_huge_page_size(void)
20+{
21+ char buf[80];
22+ int ret;
23+ int fd;
24+
25+ fd = open("/sys/kernel/mm/transparent_hugepage/hpage_pmd_size",
26+ O_RDONLY);
27+ if (fd < 0)
28+ return 2 * 1024 * 1024;
29+
30+ ret = read(fd, buf, sizeof(buf));
31+ close(fd);
32+ if (ret <= 0 || ret == sizeof(buf))
33+ return 2 * 1024 * 1024;
34+ buf[ret] = 0;
35+ return strtoul(buf, NULL, 10);
36+}
37+
38+static __attribute__((constructor)) void setup_sizes(void)
39+{
40+ void *vrc;
41+ int rc;
42+
43+ PAGE_SIZE = sysconf(_SC_PAGE_SIZE);
44+ HUGEPAGE_SIZE = get_huge_page_size();
45+
46+ BUFFER_SIZE = PAGE_SIZE * 16;
47+ rc = posix_memalign(&buffer, HUGEPAGE_SIZE, BUFFER_SIZE);
48+ assert(!rc);
49+ assert(buffer);
50+ assert((uintptr_t)buffer % HUGEPAGE_SIZE == 0);
51+ vrc = mmap(buffer, BUFFER_SIZE, PROT_READ | PROT_WRITE,
52+ MAP_SHARED | MAP_ANONYMOUS | MAP_FIXED, -1, 0);
53+ assert(vrc == buffer);
54+}
55+
56+FIXTURE(iommufd)
57+{
58+ int fd;
59+};
60+
61+FIXTURE_SETUP(iommufd)
62+{
63+ self->fd = open("/dev/iommu", O_RDWR);
64+ ASSERT_NE(-1, self->fd);
65+}
66+
67+FIXTURE_TEARDOWN(iommufd)
68+{
69+ teardown_iommufd(self->fd, _metadata);
70+}
71+
72+TEST_F(iommufd, simple_close)
73+{
74+}
75+
76+TEST_F(iommufd, cmd_fail)
77+{
78+ struct iommu_destroy cmd = { .size = sizeof(cmd), .id = 0 };
79+
80+ /* object id is invalid */
81+ EXPECT_ERRNO(ENOENT, _test_ioctl_destroy(self->fd, 0));
82+ /* Bad pointer */
83+ EXPECT_ERRNO(EFAULT, ioctl(self->fd, IOMMU_DESTROY, NULL));
84+ /* Unknown ioctl */
85+ EXPECT_ERRNO(ENOTTY,
86+ ioctl(self->fd, _IO(IOMMUFD_TYPE, IOMMUFD_CMD_BASE - 1),
87+ &cmd));
88+}
89+
90+TEST_F(iommufd, cmd_length)
91+{
92+#define TEST_LENGTH(_struct, _ioctl) \
93+ { \
94+ struct { \
95+ struct _struct cmd; \
96+ uint8_t extra; \
97+ } cmd = { .cmd = { .size = sizeof(struct _struct) - 1 }, \
98+ .extra = UINT8_MAX }; \
99+ int old_errno; \
100+ int rc; \
101+ \
102+ EXPECT_ERRNO(EINVAL, ioctl(self->fd, _ioctl, &cmd)); \
103+ cmd.cmd.size = sizeof(struct _struct) + 1; \
104+ EXPECT_ERRNO(E2BIG, ioctl(self->fd, _ioctl, &cmd)); \
105+ cmd.cmd.size = sizeof(struct _struct); \
106+ rc = ioctl(self->fd, _ioctl, &cmd); \
107+ old_errno = errno; \
108+ cmd.cmd.size = sizeof(struct _struct) + 1; \
109+ cmd.extra = 0; \
110+ if (rc) { \
111+ EXPECT_ERRNO(old_errno, \
112+ ioctl(self->fd, _ioctl, &cmd)); \
113+ } else { \
114+ ASSERT_EQ(0, ioctl(self->fd, _ioctl, &cmd)); \
115+ } \
116+ }
117+
118+ TEST_LENGTH(iommu_destroy, IOMMU_DESTROY);
119+ TEST_LENGTH(iommu_ioas_alloc, IOMMU_IOAS_ALLOC);
120+ TEST_LENGTH(iommu_ioas_iova_ranges, IOMMU_IOAS_IOVA_RANGES);
121+ TEST_LENGTH(iommu_ioas_allow_iovas, IOMMU_IOAS_ALLOW_IOVAS);
122+ TEST_LENGTH(iommu_ioas_map, IOMMU_IOAS_MAP);
123+ TEST_LENGTH(iommu_ioas_copy, IOMMU_IOAS_COPY);
124+ TEST_LENGTH(iommu_ioas_unmap, IOMMU_IOAS_UNMAP);
125+ TEST_LENGTH(iommu_option, IOMMU_OPTION);
126+ TEST_LENGTH(iommu_vfio_ioas, IOMMU_VFIO_IOAS);
127+#undef TEST_LENGTH
128+}
129+
130+TEST_F(iommufd, cmd_ex_fail)
131+{
132+ struct {
133+ struct iommu_destroy cmd;
134+ __u64 future;
135+ } cmd = { .cmd = { .size = sizeof(cmd), .id = 0 } };
136+
137+ /* object id is invalid and command is longer */
138+ EXPECT_ERRNO(ENOENT, ioctl(self->fd, IOMMU_DESTROY, &cmd));
139+ /* future area is non-zero */
140+ cmd.future = 1;
141+ EXPECT_ERRNO(E2BIG, ioctl(self->fd, IOMMU_DESTROY, &cmd));
142+ /* Original command "works" */
143+ cmd.cmd.size = sizeof(cmd.cmd);
144+ EXPECT_ERRNO(ENOENT, ioctl(self->fd, IOMMU_DESTROY, &cmd));
145+ /* Short command fails */
146+ cmd.cmd.size = sizeof(cmd.cmd) - 1;
147+ EXPECT_ERRNO(EINVAL, ioctl(self->fd, IOMMU_DESTROY, &cmd));
148+}
149+
150+TEST_F(iommufd, global_options)
151+{
152+ struct iommu_option cmd = {
153+ .size = sizeof(cmd),
154+ .option_id = IOMMU_OPTION_RLIMIT_MODE,
155+ .op = IOMMU_OPTION_OP_GET,
156+ .val64 = 1,
157+ };
158+
159+ cmd.option_id = IOMMU_OPTION_RLIMIT_MODE;
160+ ASSERT_EQ(0, ioctl(self->fd, IOMMU_OPTION, &cmd));
161+ ASSERT_EQ(0, cmd.val64);
162+
163+ /* This requires root */
164+ cmd.op = IOMMU_OPTION_OP_SET;
165+ cmd.val64 = 1;
166+ ASSERT_EQ(0, ioctl(self->fd, IOMMU_OPTION, &cmd));
167+ cmd.val64 = 2;
168+ EXPECT_ERRNO(EINVAL, ioctl(self->fd, IOMMU_OPTION, &cmd));
169+
170+ cmd.op = IOMMU_OPTION_OP_GET;
171+ ASSERT_EQ(0, ioctl(self->fd, IOMMU_OPTION, &cmd));
172+ ASSERT_EQ(1, cmd.val64);
173+
174+ cmd.op = IOMMU_OPTION_OP_SET;
175+ cmd.val64 = 0;
176+ ASSERT_EQ(0, ioctl(self->fd, IOMMU_OPTION, &cmd));
177+
178+ cmd.op = IOMMU_OPTION_OP_GET;
179+ cmd.option_id = IOMMU_OPTION_HUGE_PAGES;
180+ EXPECT_ERRNO(ENOENT, ioctl(self->fd, IOMMU_OPTION, &cmd));
181+ cmd.op = IOMMU_OPTION_OP_SET;
182+ EXPECT_ERRNO(ENOENT, ioctl(self->fd, IOMMU_OPTION, &cmd));
183+}
184+
185+FIXTURE(iommufd_ioas)
186+{
187+ int fd;
188+ uint32_t ioas_id;
189+ uint32_t domain_id;
190+ uint64_t base_iova;
191+};
192+
193+FIXTURE_VARIANT(iommufd_ioas)
194+{
195+ unsigned int mock_domains;
196+ unsigned int memory_limit;
197+};
198+
199+FIXTURE_SETUP(iommufd_ioas)
200+{
201+ unsigned int i;
202+
203+
204+ self->fd = open("/dev/iommu", O_RDWR);
205+ ASSERT_NE(-1, self->fd);
206+ test_ioctl_ioas_alloc(&self->ioas_id);
207+
208+ if (!variant->memory_limit) {
209+ test_ioctl_set_default_memory_limit();
210+ } else {
211+ test_ioctl_set_temp_memory_limit(variant->memory_limit);
212+ }
213+
214+ for (i = 0; i != variant->mock_domains; i++) {
215+ test_cmd_mock_domain(self->ioas_id, NULL, &self->domain_id);
216+ self->base_iova = MOCK_APERTURE_START;
217+ }
218+}
219+
220+FIXTURE_TEARDOWN(iommufd_ioas)
221+{
222+ test_ioctl_set_default_memory_limit();
223+ teardown_iommufd(self->fd, _metadata);
224+}
225+
226+FIXTURE_VARIANT_ADD(iommufd_ioas, no_domain)
227+{
228+};
229+
230+FIXTURE_VARIANT_ADD(iommufd_ioas, mock_domain)
231+{
232+ .mock_domains = 1,
233+};
234+
235+FIXTURE_VARIANT_ADD(iommufd_ioas, two_mock_domain)
236+{
237+ .mock_domains = 2,
238+};
239+
240+FIXTURE_VARIANT_ADD(iommufd_ioas, mock_domain_limit)
241+{
242+ .mock_domains = 1,
243+ .memory_limit = 16,
244+};
245+
246+TEST_F(iommufd_ioas, ioas_auto_destroy)
247+{
248+}
249+
250+TEST_F(iommufd_ioas, ioas_destroy)
251+{
252+ if (self->domain_id) {
253+ /* IOAS cannot be freed while a domain is on it */
254+ EXPECT_ERRNO(EBUSY,
255+ _test_ioctl_destroy(self->fd, self->ioas_id));
256+ } else {
257+ /* Can allocate and manually free an IOAS table */
258+ test_ioctl_destroy(self->ioas_id);
259+ }
260+}
261+
262+TEST_F(iommufd_ioas, ioas_area_destroy)
263+{
264+ /* Adding an area does not change ability to destroy */
265+ test_ioctl_ioas_map_fixed(buffer, PAGE_SIZE, self->base_iova);
266+ if (self->domain_id)
267+ EXPECT_ERRNO(EBUSY,
268+ _test_ioctl_destroy(self->fd, self->ioas_id));
269+ else
270+ test_ioctl_destroy(self->ioas_id);
271+}
272+
273+TEST_F(iommufd_ioas, ioas_area_auto_destroy)
274+{
275+ int i;
276+
277+ /* Can allocate and automatically free an IOAS table with many areas */
278+ for (i = 0; i != 10; i++) {
279+ test_ioctl_ioas_map_fixed(buffer, PAGE_SIZE,
280+ self->base_iova + i * PAGE_SIZE);
281+ }
282+}
283+
284+TEST_F(iommufd_ioas, area)
285+{
286+ int i;
287+
288+ /* Unmap fails if nothing is mapped */
289+ for (i = 0; i != 10; i++)
290+ test_err_ioctl_ioas_unmap(ENOENT, i * PAGE_SIZE, PAGE_SIZE);
291+
292+ /* Unmap works */
293+ for (i = 0; i != 10; i++)
294+ test_ioctl_ioas_map_fixed(buffer, PAGE_SIZE,
295+ self->base_iova + i * PAGE_SIZE);
296+ for (i = 0; i != 10; i++)
297+ test_ioctl_ioas_unmap(self->base_iova + i * PAGE_SIZE,
298+ PAGE_SIZE);
299+
300+ /* Split fails */
301+ test_ioctl_ioas_map_fixed(buffer, PAGE_SIZE * 2,
302+ self->base_iova + 16 * PAGE_SIZE);
303+ test_err_ioctl_ioas_unmap(ENOENT, self->base_iova + 16 * PAGE_SIZE,
304+ PAGE_SIZE);
305+ test_err_ioctl_ioas_unmap(ENOENT, self->base_iova + 17 * PAGE_SIZE,
306+ PAGE_SIZE);
307+
308+ /* Over map fails */
309+ test_err_ioctl_ioas_map_fixed(EEXIST, buffer, PAGE_SIZE * 2,
310+ self->base_iova + 16 * PAGE_SIZE);
311+ test_err_ioctl_ioas_map_fixed(EEXIST, buffer, PAGE_SIZE,
312+ self->base_iova + 16 * PAGE_SIZE);
313+ test_err_ioctl_ioas_map_fixed(EEXIST, buffer, PAGE_SIZE,
314+ self->base_iova + 17 * PAGE_SIZE);
315+ test_err_ioctl_ioas_map_fixed(EEXIST, buffer, PAGE_SIZE * 2,
316+ self->base_iova + 15 * PAGE_SIZE);
317+ test_err_ioctl_ioas_map_fixed(EEXIST, buffer, PAGE_SIZE * 3,
318+ self->base_iova + 15 * PAGE_SIZE);
319+
320+ /* unmap all works */
321+ test_ioctl_ioas_unmap(0, UINT64_MAX);
322+
323+ /* Unmap all succeeds on an empty IOAS */
324+ test_ioctl_ioas_unmap(0, UINT64_MAX);
325+}
326+
327+TEST_F(iommufd_ioas, unmap_fully_contained_areas)
328+{
329+ uint64_t unmap_len;
330+ int i;
331+
332+ /* Give no_domain some space to rewind base_iova */
333+ self->base_iova += 4 * PAGE_SIZE;
334+
335+ for (i = 0; i != 4; i++)
336+ test_ioctl_ioas_map_fixed(buffer, 8 * PAGE_SIZE,
337+ self->base_iova + i * 16 * PAGE_SIZE);
338+
339+ /* Unmap not fully contained area doesn't work */
340+ test_err_ioctl_ioas_unmap(ENOENT, self->base_iova - 4 * PAGE_SIZE,
341+ 8 * PAGE_SIZE);
342+ test_err_ioctl_ioas_unmap(ENOENT,
343+ self->base_iova + 3 * 16 * PAGE_SIZE +
344+ 8 * PAGE_SIZE - 4 * PAGE_SIZE,
345+ 8 * PAGE_SIZE);
346+
347+ /* Unmap fully contained areas works */
348+ ASSERT_EQ(0, _test_ioctl_ioas_unmap(self->fd, self->ioas_id,
349+ self->base_iova - 4 * PAGE_SIZE,
350+ 3 * 16 * PAGE_SIZE + 8 * PAGE_SIZE +
351+ 4 * PAGE_SIZE,
352+ &unmap_len));
353+ ASSERT_EQ(32 * PAGE_SIZE, unmap_len);
354+}
355+
356+TEST_F(iommufd_ioas, area_auto_iova)
357+{
358+ struct iommu_test_cmd test_cmd = {
359+ .size = sizeof(test_cmd),
360+ .op = IOMMU_TEST_OP_ADD_RESERVED,
361+ .id = self->ioas_id,
362+ .add_reserved = { .start = PAGE_SIZE * 4,
363+ .length = PAGE_SIZE * 100 },
364+ };
365+ struct iommu_iova_range ranges[1] = {};
366+ struct iommu_ioas_allow_iovas allow_cmd = {
367+ .size = sizeof(allow_cmd),
368+ .ioas_id = self->ioas_id,
369+ .num_iovas = 1,
370+ .allowed_iovas = (uintptr_t)ranges,
371+ };
372+ __u64 iovas[10];
373+ int i;
374+
375+ /* Simple 4k pages */
376+ for (i = 0; i != 10; i++)
377+ test_ioctl_ioas_map(buffer, PAGE_SIZE, &iovas[i]);
378+ for (i = 0; i != 10; i++)
379+ test_ioctl_ioas_unmap(iovas[i], PAGE_SIZE);
380+
381+ /* Kernel automatically aligns IOVAs properly */
382+ for (i = 0; i != 10; i++) {
383+ size_t length = PAGE_SIZE * (i + 1);
384+
385+ if (self->domain_id) {
386+ test_ioctl_ioas_map(buffer, length, &iovas[i]);
387+ } else {
388+ test_ioctl_ioas_map((void *)(1UL << 31), length,
389+ &iovas[i]);
390+ }
391+ EXPECT_EQ(0, iovas[i] % (1UL << (ffs(length) - 1)));
392+ }
393+ for (i = 0; i != 10; i++)
394+ test_ioctl_ioas_unmap(iovas[i], PAGE_SIZE * (i + 1));
395+
396+ /* Avoids a reserved region */
397+ ASSERT_EQ(0,
398+ ioctl(self->fd, _IOMMU_TEST_CMD(IOMMU_TEST_OP_ADD_RESERVED),
399+ &test_cmd));
400+ for (i = 0; i != 10; i++) {
401+ size_t length = PAGE_SIZE * (i + 1);
402+
403+ test_ioctl_ioas_map(buffer, length, &iovas[i]);
404+ EXPECT_EQ(0, iovas[i] % (1UL << (ffs(length) - 1)));
405+ EXPECT_EQ(false,
406+ iovas[i] > test_cmd.add_reserved.start &&
407+ iovas[i] <
408+ test_cmd.add_reserved.start +
409+ test_cmd.add_reserved.length);
410+ }
411+ for (i = 0; i != 10; i++)
412+ test_ioctl_ioas_unmap(iovas[i], PAGE_SIZE * (i + 1));
413+
414+ /* Allowed region intersects with a reserved region */
415+ ranges[0].start = PAGE_SIZE;
416+ ranges[0].last = PAGE_SIZE * 600;
417+ EXPECT_ERRNO(EADDRINUSE,
418+ ioctl(self->fd, IOMMU_IOAS_ALLOW_IOVAS, &allow_cmd));
419+
420+ /* Allocate from an allowed region */
421+ if (self->domain_id) {
422+ ranges[0].start = MOCK_APERTURE_START + PAGE_SIZE;
423+ ranges[0].last = MOCK_APERTURE_START + PAGE_SIZE * 600 - 1;
424+ } else {
425+ ranges[0].start = PAGE_SIZE * 200;
426+ ranges[0].last = PAGE_SIZE * 600 - 1;
427+ }
428+ ASSERT_EQ(0, ioctl(self->fd, IOMMU_IOAS_ALLOW_IOVAS, &allow_cmd));
429+ for (i = 0; i != 10; i++) {
430+ size_t length = PAGE_SIZE * (i + 1);
431+
432+ test_ioctl_ioas_map(buffer, length, &iovas[i]);
433+ EXPECT_EQ(0, iovas[i] % (1UL << (ffs(length) - 1)));
434+ EXPECT_EQ(true, iovas[i] >= ranges[0].start);
435+ EXPECT_EQ(true, iovas[i] <= ranges[0].last);
436+ EXPECT_EQ(true, iovas[i] + length > ranges[0].start);
437+ EXPECT_EQ(true, iovas[i] + length <= ranges[0].last + 1);
438+ }
439+ for (i = 0; i != 10; i++)
440+ test_ioctl_ioas_unmap(iovas[i], PAGE_SIZE * (i + 1));
441+}
442+
443+TEST_F(iommufd_ioas, area_allowed)
444+{
445+ struct iommu_test_cmd test_cmd = {
446+ .size = sizeof(test_cmd),
447+ .op = IOMMU_TEST_OP_ADD_RESERVED,
448+ .id = self->ioas_id,
449+ .add_reserved = { .start = PAGE_SIZE * 4,
450+ .length = PAGE_SIZE * 100 },
451+ };
452+ struct iommu_iova_range ranges[1] = {};
453+ struct iommu_ioas_allow_iovas allow_cmd = {
454+ .size = sizeof(allow_cmd),
455+ .ioas_id = self->ioas_id,
456+ .num_iovas = 1,
457+ .allowed_iovas = (uintptr_t)ranges,
458+ };
459+
460+ /* Reserved intersects an allowed */
461+ allow_cmd.num_iovas = 1;
462+ ranges[0].start = self->base_iova;
463+ ranges[0].last = ranges[0].start + PAGE_SIZE * 600;
464+ ASSERT_EQ(0, ioctl(self->fd, IOMMU_IOAS_ALLOW_IOVAS, &allow_cmd));
465+ test_cmd.add_reserved.start = ranges[0].start + PAGE_SIZE;
466+ test_cmd.add_reserved.length = PAGE_SIZE;
467+ EXPECT_ERRNO(EADDRINUSE,
468+ ioctl(self->fd,
469+ _IOMMU_TEST_CMD(IOMMU_TEST_OP_ADD_RESERVED),
470+ &test_cmd));
471+ allow_cmd.num_iovas = 0;
472+ ASSERT_EQ(0, ioctl(self->fd, IOMMU_IOAS_ALLOW_IOVAS, &allow_cmd));
473+
474+ /* Allowed intersects a reserved */
475+ ASSERT_EQ(0,
476+ ioctl(self->fd, _IOMMU_TEST_CMD(IOMMU_TEST_OP_ADD_RESERVED),
477+ &test_cmd));
478+ allow_cmd.num_iovas = 1;
479+ ranges[0].start = self->base_iova;
480+ ranges[0].last = ranges[0].start + PAGE_SIZE * 600;
481+ EXPECT_ERRNO(EADDRINUSE,
482+ ioctl(self->fd, IOMMU_IOAS_ALLOW_IOVAS, &allow_cmd));
483+}
484+
485+TEST_F(iommufd_ioas, copy_area)
486+{
487+ struct iommu_ioas_copy copy_cmd = {
488+ .size = sizeof(copy_cmd),
489+ .flags = IOMMU_IOAS_MAP_FIXED_IOVA,
490+ .dst_ioas_id = self->ioas_id,
491+ .src_ioas_id = self->ioas_id,
492+ .length = PAGE_SIZE,
493+ };
494+
495+ test_ioctl_ioas_map_fixed(buffer, PAGE_SIZE, self->base_iova);
496+
497+ /* Copy inside a single IOAS */
498+ copy_cmd.src_iova = self->base_iova;
499+ copy_cmd.dst_iova = self->base_iova + PAGE_SIZE;
500+ ASSERT_EQ(0, ioctl(self->fd, IOMMU_IOAS_COPY, &copy_cmd));
501+
502+ /* Copy between IOAS's */
503+ copy_cmd.src_iova = self->base_iova;
504+ copy_cmd.dst_iova = 0;
505+ test_ioctl_ioas_alloc(&copy_cmd.dst_ioas_id);
506+ ASSERT_EQ(0, ioctl(self->fd, IOMMU_IOAS_COPY, &copy_cmd));
507+}
508+
509+TEST_F(iommufd_ioas, iova_ranges)
510+{
511+ struct iommu_test_cmd test_cmd = {
512+ .size = sizeof(test_cmd),
513+ .op = IOMMU_TEST_OP_ADD_RESERVED,
514+ .id = self->ioas_id,
515+ .add_reserved = { .start = PAGE_SIZE, .length = PAGE_SIZE },
516+ };
517+ struct iommu_iova_range *ranges = buffer;
518+ struct iommu_ioas_iova_ranges ranges_cmd = {
519+ .size = sizeof(ranges_cmd),
520+ .ioas_id = self->ioas_id,
521+ .num_iovas = BUFFER_SIZE / sizeof(*ranges),
522+ .allowed_iovas = (uintptr_t)ranges,
523+ };
524+
525+ /* Range can be read */
526+ ASSERT_EQ(0, ioctl(self->fd, IOMMU_IOAS_IOVA_RANGES, &ranges_cmd));
527+ EXPECT_EQ(1, ranges_cmd.num_iovas);
528+ if (!self->domain_id) {
529+ EXPECT_EQ(0, ranges[0].start);
530+ EXPECT_EQ(SIZE_MAX, ranges[0].last);
531+ EXPECT_EQ(1, ranges_cmd.out_iova_alignment);
532+ } else {
533+ EXPECT_EQ(MOCK_APERTURE_START, ranges[0].start);
534+ EXPECT_EQ(MOCK_APERTURE_LAST, ranges[0].last);
535+ EXPECT_EQ(MOCK_PAGE_SIZE, ranges_cmd.out_iova_alignment);
536+ }
537+
538+ /* Buffer too small */
539+ memset(ranges, 0, BUFFER_SIZE);
540+ ranges_cmd.num_iovas = 0;
541+ EXPECT_ERRNO(EMSGSIZE,
542+ ioctl(self->fd, IOMMU_IOAS_IOVA_RANGES, &ranges_cmd));
543+ EXPECT_EQ(1, ranges_cmd.num_iovas);
544+ EXPECT_EQ(0, ranges[0].start);
545+ EXPECT_EQ(0, ranges[0].last);
546+
547+ /* 2 ranges */
548+ ASSERT_EQ(0,
549+ ioctl(self->fd, _IOMMU_TEST_CMD(IOMMU_TEST_OP_ADD_RESERVED),
550+ &test_cmd));
551+ ranges_cmd.num_iovas = BUFFER_SIZE / sizeof(*ranges);
552+ ASSERT_EQ(0, ioctl(self->fd, IOMMU_IOAS_IOVA_RANGES, &ranges_cmd));
553+ if (!self->domain_id) {
554+ EXPECT_EQ(2, ranges_cmd.num_iovas);
555+ EXPECT_EQ(0, ranges[0].start);
556+ EXPECT_EQ(PAGE_SIZE - 1, ranges[0].last);
557+ EXPECT_EQ(PAGE_SIZE * 2, ranges[1].start);
558+ EXPECT_EQ(SIZE_MAX, ranges[1].last);
559+ } else {
560+ EXPECT_EQ(1, ranges_cmd.num_iovas);
561+ EXPECT_EQ(MOCK_APERTURE_START, ranges[0].start);
562+ EXPECT_EQ(MOCK_APERTURE_LAST, ranges[0].last);
563+ }
564+
565+ /* Buffer too small */
566+ memset(ranges, 0, BUFFER_SIZE);
567+ ranges_cmd.num_iovas = 1;
568+ if (!self->domain_id) {
569+ EXPECT_ERRNO(EMSGSIZE, ioctl(self->fd, IOMMU_IOAS_IOVA_RANGES,
570+ &ranges_cmd));
571+ EXPECT_EQ(2, ranges_cmd.num_iovas);
572+ EXPECT_EQ(0, ranges[0].start);
573+ EXPECT_EQ(PAGE_SIZE - 1, ranges[0].last);
574+ } else {
575+ ASSERT_EQ(0,
576+ ioctl(self->fd, IOMMU_IOAS_IOVA_RANGES, &ranges_cmd));
577+ EXPECT_EQ(1, ranges_cmd.num_iovas);
578+ EXPECT_EQ(MOCK_APERTURE_START, ranges[0].start);
579+ EXPECT_EQ(MOCK_APERTURE_LAST, ranges[0].last);
580+ }
581+ EXPECT_EQ(0, ranges[1].start);
582+ EXPECT_EQ(0, ranges[1].last);
583+}
584+
585+TEST_F(iommufd_ioas, access_pin)
586+{
587+ struct iommu_test_cmd access_cmd = {
588+ .size = sizeof(access_cmd),
589+ .op = IOMMU_TEST_OP_ACCESS_PAGES,
590+ .access_pages = { .iova = MOCK_APERTURE_START,
591+ .length = BUFFER_SIZE,
592+ .uptr = (uintptr_t)buffer },
593+ };
594+ struct iommu_test_cmd check_map_cmd = {
595+ .size = sizeof(check_map_cmd),
596+ .op = IOMMU_TEST_OP_MD_CHECK_MAP,
597+ .check_map = { .iova = MOCK_APERTURE_START,
598+ .length = BUFFER_SIZE,
599+ .uptr = (uintptr_t)buffer },
600+ };
601+ uint32_t access_pages_id;
602+ unsigned int npages;
603+
604+ test_cmd_create_access(self->ioas_id, &access_cmd.id,
605+ MOCK_FLAGS_ACCESS_CREATE_NEEDS_PIN_PAGES);
606+
607+ for (npages = 1; npages < BUFFER_SIZE / PAGE_SIZE; npages++) {
608+ uint32_t mock_device_id;
609+ uint32_t mock_hwpt_id;
610+
611+ access_cmd.access_pages.length = npages * PAGE_SIZE;
612+
613+ /* Single map/unmap */
614+ test_ioctl_ioas_map_fixed(buffer, BUFFER_SIZE,
615+ MOCK_APERTURE_START);
616+ ASSERT_EQ(0, ioctl(self->fd,
617+ _IOMMU_TEST_CMD(IOMMU_TEST_OP_ACCESS_PAGES),
618+ &access_cmd));
619+ test_cmd_destroy_access_pages(
620+ access_cmd.id,
621+ access_cmd.access_pages.out_access_pages_id);
622+
623+ /* Double user */
624+ ASSERT_EQ(0, ioctl(self->fd,
625+ _IOMMU_TEST_CMD(IOMMU_TEST_OP_ACCESS_PAGES),
626+ &access_cmd));
627+ access_pages_id = access_cmd.access_pages.out_access_pages_id;
628+ ASSERT_EQ(0, ioctl(self->fd,
629+ _IOMMU_TEST_CMD(IOMMU_TEST_OP_ACCESS_PAGES),
630+ &access_cmd));
631+ test_cmd_destroy_access_pages(
632+ access_cmd.id,
633+ access_cmd.access_pages.out_access_pages_id);
634+ test_cmd_destroy_access_pages(access_cmd.id, access_pages_id);
635+
636+ /* Add/remove a domain with a user */
637+ ASSERT_EQ(0, ioctl(self->fd,
638+ _IOMMU_TEST_CMD(IOMMU_TEST_OP_ACCESS_PAGES),
639+ &access_cmd));
640+ test_cmd_mock_domain(self->ioas_id, &mock_device_id,
641+ &mock_hwpt_id);
642+ check_map_cmd.id = mock_hwpt_id;
643+ ASSERT_EQ(0, ioctl(self->fd,
644+ _IOMMU_TEST_CMD(IOMMU_TEST_OP_MD_CHECK_MAP),
645+ &check_map_cmd));
646+
647+ test_ioctl_destroy(mock_device_id);
648+ test_ioctl_destroy(mock_hwpt_id);
649+ test_cmd_destroy_access_pages(
650+ access_cmd.id,
651+ access_cmd.access_pages.out_access_pages_id);
652+
653+ test_ioctl_ioas_unmap(MOCK_APERTURE_START, BUFFER_SIZE);
654+ }
655+ test_cmd_destroy_access(access_cmd.id);
656+}
657+
658+TEST_F(iommufd_ioas, access_pin_unmap)
659+{
660+ struct iommu_test_cmd access_pages_cmd = {
661+ .size = sizeof(access_pages_cmd),
662+ .op = IOMMU_TEST_OP_ACCESS_PAGES,
663+ .access_pages = { .iova = MOCK_APERTURE_START,
664+ .length = BUFFER_SIZE,
665+ .uptr = (uintptr_t)buffer },
666+ };
667+
668+ test_cmd_create_access(self->ioas_id, &access_pages_cmd.id,
669+ MOCK_FLAGS_ACCESS_CREATE_NEEDS_PIN_PAGES);
670+ test_ioctl_ioas_map_fixed(buffer, BUFFER_SIZE, MOCK_APERTURE_START);
671+ ASSERT_EQ(0,
672+ ioctl(self->fd, _IOMMU_TEST_CMD(IOMMU_TEST_OP_ACCESS_PAGES),
673+ &access_pages_cmd));
674+
675+ /* Trigger the unmap op */
676+ test_ioctl_ioas_unmap(MOCK_APERTURE_START, BUFFER_SIZE);
677+
678+ /* kernel removed the item for us */
679+ test_err_destroy_access_pages(
680+ ENOENT, access_pages_cmd.id,
681+ access_pages_cmd.access_pages.out_access_pages_id);
682+}
683+
684+static void check_access_rw(struct __test_metadata *_metadata, int fd,
685+ unsigned int access_id, uint64_t iova,
686+ unsigned int def_flags)
687+{
688+ uint16_t tmp[32];
689+ struct iommu_test_cmd access_cmd = {
690+ .size = sizeof(access_cmd),
691+ .op = IOMMU_TEST_OP_ACCESS_RW,
692+ .id = access_id,
693+ .access_rw = { .uptr = (uintptr_t)tmp },
694+ };
695+ uint16_t *buffer16 = buffer;
696+ unsigned int i;
697+ void *tmp2;
698+
699+ for (i = 0; i != BUFFER_SIZE / sizeof(*buffer16); i++)
700+ buffer16[i] = rand();
701+
702+ for (access_cmd.access_rw.iova = iova + PAGE_SIZE - 50;
703+ access_cmd.access_rw.iova < iova + PAGE_SIZE + 50;
704+ access_cmd.access_rw.iova++) {
705+ for (access_cmd.access_rw.length = 1;
706+ access_cmd.access_rw.length < sizeof(tmp);
707+ access_cmd.access_rw.length++) {
708+ access_cmd.access_rw.flags = def_flags;
709+ ASSERT_EQ(0, ioctl(fd,
710+ _IOMMU_TEST_CMD(
711+ IOMMU_TEST_OP_ACCESS_RW),
712+ &access_cmd));
713+ ASSERT_EQ(0,
714+ memcmp(buffer + (access_cmd.access_rw.iova -
715+ iova),
716+ tmp, access_cmd.access_rw.length));
717+
718+ for (i = 0; i != ARRAY_SIZE(tmp); i++)
719+ tmp[i] = rand();
720+ access_cmd.access_rw.flags = def_flags |
721+ MOCK_ACCESS_RW_WRITE;
722+ ASSERT_EQ(0, ioctl(fd,
723+ _IOMMU_TEST_CMD(
724+ IOMMU_TEST_OP_ACCESS_RW),
725+ &access_cmd));
726+ ASSERT_EQ(0,
727+ memcmp(buffer + (access_cmd.access_rw.iova -
728+ iova),
729+ tmp, access_cmd.access_rw.length));
730+ }
731+ }
732+
733+ /* Multi-page test */
734+ tmp2 = malloc(BUFFER_SIZE);
735+ ASSERT_NE(NULL, tmp2);
736+ access_cmd.access_rw.iova = iova;
737+ access_cmd.access_rw.length = BUFFER_SIZE;
738+ access_cmd.access_rw.flags = def_flags;
739+ access_cmd.access_rw.uptr = (uintptr_t)tmp2;
740+ ASSERT_EQ(0, ioctl(fd, _IOMMU_TEST_CMD(IOMMU_TEST_OP_ACCESS_RW),
741+ &access_cmd));
742+ ASSERT_EQ(0, memcmp(buffer, tmp2, access_cmd.access_rw.length));
743+ free(tmp2);
744+}
745+
746+TEST_F(iommufd_ioas, access_rw)
747+{
748+ __u32 access_id;
749+ __u64 iova;
750+
751+ test_cmd_create_access(self->ioas_id, &access_id, 0);
752+ test_ioctl_ioas_map(buffer, BUFFER_SIZE, &iova);
753+ check_access_rw(_metadata, self->fd, access_id, iova, 0);
754+ check_access_rw(_metadata, self->fd, access_id, iova,
755+ MOCK_ACCESS_RW_SLOW_PATH);
756+ test_ioctl_ioas_unmap(iova, BUFFER_SIZE);
757+ test_cmd_destroy_access(access_id);
758+}
759+
760+TEST_F(iommufd_ioas, access_rw_unaligned)
761+{
762+ __u32 access_id;
763+ __u64 iova;
764+
765+ test_cmd_create_access(self->ioas_id, &access_id, 0);
766+
767+ /* Unaligned pages */
768+ iova = self->base_iova + MOCK_PAGE_SIZE;
769+ test_ioctl_ioas_map_fixed(buffer, BUFFER_SIZE, iova);
770+ check_access_rw(_metadata, self->fd, access_id, iova, 0);
771+ test_ioctl_ioas_unmap(iova, BUFFER_SIZE);
772+ test_cmd_destroy_access(access_id);
773+}
774+
775+TEST_F(iommufd_ioas, fork_gone)
776+{
777+ __u32 access_id;
778+ pid_t child;
779+
780+ test_cmd_create_access(self->ioas_id, &access_id, 0);
781+
782+ /* Create a mapping with a different mm */
783+ child = fork();
784+ if (!child) {
785+ test_ioctl_ioas_map_fixed(buffer, BUFFER_SIZE,
786+ MOCK_APERTURE_START);
787+ exit(0);
788+ }
789+ ASSERT_NE(-1, child);
790+ ASSERT_EQ(child, waitpid(child, NULL, 0));
791+
792+ if (self->domain_id) {
793+ /*
794+ * If a domain already existed then everything was pinned within
795+ * the fork, so this copies from one domain to another.
796+ */
797+ test_cmd_mock_domain(self->ioas_id, NULL, NULL);
798+ check_access_rw(_metadata, self->fd, access_id,
799+ MOCK_APERTURE_START, 0);
800+
801+ } else {
802+ /*
803+ * Otherwise we need to actually pin pages which can't happen
804+ * since the fork is gone.
805+ */
806+ test_err_mock_domain(EFAULT, self->ioas_id, NULL, NULL);
807+ }
808+
809+ test_cmd_destroy_access(access_id);
810+}
811+
812+TEST_F(iommufd_ioas, fork_present)
813+{
814+ __u32 access_id;
815+ int pipefds[2];
816+ uint64_t tmp;
817+ pid_t child;
818+ int efd;
819+
820+ test_cmd_create_access(self->ioas_id, &access_id, 0);
821+
822+ ASSERT_EQ(0, pipe2(pipefds, O_CLOEXEC));
823+ efd = eventfd(0, EFD_CLOEXEC);
824+ ASSERT_NE(-1, efd);
825+
826+ /* Create a mapping with a different mm */
827+ child = fork();
828+ if (!child) {
829+ __u64 iova;
830+ uint64_t one = 1;
831+
832+ close(pipefds[1]);
833+ test_ioctl_ioas_map_fixed(buffer, BUFFER_SIZE,
834+ MOCK_APERTURE_START);
835+ if (write(efd, &one, sizeof(one)) != sizeof(one))
836+ exit(100);
837+ if (read(pipefds[0], &iova, 1) != 1)
838+ exit(100);
839+ exit(0);
840+ }
841+ close(pipefds[0]);
842+ ASSERT_NE(-1, child);
843+ ASSERT_EQ(8, read(efd, &tmp, sizeof(tmp)));
844+
845+ /* Read pages from the remote process */
846+ test_cmd_mock_domain(self->ioas_id, NULL, NULL);
847+ check_access_rw(_metadata, self->fd, access_id, MOCK_APERTURE_START, 0);
848+
849+ ASSERT_EQ(0, close(pipefds[1]));
850+ ASSERT_EQ(child, waitpid(child, NULL, 0));
851+
852+ test_cmd_destroy_access(access_id);
853+}
854+
855+TEST_F(iommufd_ioas, ioas_option_huge_pages)
856+{
857+ struct iommu_option cmd = {
858+ .size = sizeof(cmd),
859+ .option_id = IOMMU_OPTION_HUGE_PAGES,
860+ .op = IOMMU_OPTION_OP_GET,
861+ .val64 = 3,
862+ .object_id = self->ioas_id,
863+ };
864+
865+ ASSERT_EQ(0, ioctl(self->fd, IOMMU_OPTION, &cmd));
866+ ASSERT_EQ(1, cmd.val64);
867+
868+ cmd.op = IOMMU_OPTION_OP_SET;
869+ cmd.val64 = 0;
870+ ASSERT_EQ(0, ioctl(self->fd, IOMMU_OPTION, &cmd));
871+
872+ cmd.op = IOMMU_OPTION_OP_GET;
873+ cmd.val64 = 3;
874+ ASSERT_EQ(0, ioctl(self->fd, IOMMU_OPTION, &cmd));
875+ ASSERT_EQ(0, cmd.val64);
876+
877+ cmd.op = IOMMU_OPTION_OP_SET;
878+ cmd.val64 = 2;
879+ EXPECT_ERRNO(EINVAL, ioctl(self->fd, IOMMU_OPTION, &cmd));
880+
881+ cmd.op = IOMMU_OPTION_OP_SET;
882+ cmd.val64 = 1;
883+ ASSERT_EQ(0, ioctl(self->fd, IOMMU_OPTION, &cmd));
884+}
885+
886+TEST_F(iommufd_ioas, ioas_iova_alloc)
887+{
888+ unsigned int length;
889+ __u64 iova;
890+
891+ for (length = 1; length != PAGE_SIZE * 2; length++) {
892+ if (variant->mock_domains && (length % MOCK_PAGE_SIZE)) {
893+ test_err_ioctl_ioas_map(EINVAL, buffer, length, &iova);
894+ } else {
895+ test_ioctl_ioas_map(buffer, length, &iova);
896+ test_ioctl_ioas_unmap(iova, length);
897+ }
898+ }
899+}
900+
901+TEST_F(iommufd_ioas, ioas_align_change)
902+{
903+ struct iommu_option cmd = {
904+ .size = sizeof(cmd),
905+ .option_id = IOMMU_OPTION_HUGE_PAGES,
906+ .op = IOMMU_OPTION_OP_SET,
907+ .object_id = self->ioas_id,
908+ /* 0 means everything must be aligned to PAGE_SIZE */
909+ .val64 = 0,
910+ };
911+
912+ /*
913+ * We cannot upgrade the alignment using OPTION_HUGE_PAGES when a domain
914+ * and map are present.
915+ */
916+ if (variant->mock_domains)
917+ return;
918+
919+ /*
920+ * We can upgrade to PAGE_SIZE alignment when things are aligned right
921+ */
922+ test_ioctl_ioas_map_fixed(buffer, PAGE_SIZE, MOCK_APERTURE_START);
923+ ASSERT_EQ(0, ioctl(self->fd, IOMMU_OPTION, &cmd));
924+
925+ /* Misalignment is rejected at map time */
926+ test_err_ioctl_ioas_map_fixed(EINVAL, buffer + MOCK_PAGE_SIZE,
927+ PAGE_SIZE,
928+ MOCK_APERTURE_START + PAGE_SIZE);
929+ ASSERT_EQ(0, ioctl(self->fd, IOMMU_OPTION, &cmd));
930+
931+ /* Reduce alignment */
932+ cmd.val64 = 1;
933+ ASSERT_EQ(0, ioctl(self->fd, IOMMU_OPTION, &cmd));
934+
935+ /* Confirm misalignment is rejected during alignment upgrade */
936+ test_ioctl_ioas_map_fixed(buffer + MOCK_PAGE_SIZE, PAGE_SIZE,
937+ MOCK_APERTURE_START + PAGE_SIZE);
938+ cmd.val64 = 0;
939+ EXPECT_ERRNO(EADDRINUSE, ioctl(self->fd, IOMMU_OPTION, &cmd));
940+
941+ test_ioctl_ioas_unmap(MOCK_APERTURE_START + PAGE_SIZE, PAGE_SIZE);
942+ test_ioctl_ioas_unmap(MOCK_APERTURE_START, PAGE_SIZE);
943+}
944+
945+TEST_F(iommufd_ioas, copy_sweep)
946+{
947+ struct iommu_ioas_copy copy_cmd = {
948+ .size = sizeof(copy_cmd),
949+ .flags = IOMMU_IOAS_MAP_FIXED_IOVA,
950+ .src_ioas_id = self->ioas_id,
951+ .dst_iova = MOCK_APERTURE_START,
952+ .length = MOCK_PAGE_SIZE,
953+ };
954+ unsigned int dst_ioas_id;
955+ uint64_t last_iova;
956+ uint64_t iova;
957+
958+ test_ioctl_ioas_alloc(&dst_ioas_id);
959+ copy_cmd.dst_ioas_id = dst_ioas_id;
960+
961+ if (variant->mock_domains)
962+ last_iova = MOCK_APERTURE_START + BUFFER_SIZE - 1;
963+ else
964+ last_iova = MOCK_APERTURE_START + BUFFER_SIZE - 2;
965+
966+ test_ioctl_ioas_map_fixed(buffer, last_iova - MOCK_APERTURE_START + 1,
967+ MOCK_APERTURE_START);
968+
969+ for (iova = MOCK_APERTURE_START - PAGE_SIZE; iova <= last_iova;
970+ iova += 511) {
971+ copy_cmd.src_iova = iova;
972+ if (iova < MOCK_APERTURE_START ||
973+ iova + copy_cmd.length - 1 > last_iova) {
974+ EXPECT_ERRNO(ENOENT, ioctl(self->fd, IOMMU_IOAS_COPY,
975+ &copy_cmd));
976+ } else {
977+ ASSERT_EQ(0,
978+ ioctl(self->fd, IOMMU_IOAS_COPY, &copy_cmd));
979+ test_ioctl_ioas_unmap_id(dst_ioas_id, copy_cmd.dst_iova,
980+ copy_cmd.length);
981+ }
982+ }
983+
984+ test_ioctl_destroy(dst_ioas_id);
985+}
986+
987+FIXTURE(iommufd_mock_domain)
988+{
989+ int fd;
990+ uint32_t ioas_id;
991+ uint32_t domain_id;
992+ uint32_t domain_ids[2];
993+ int mmap_flags;
994+ size_t mmap_buf_size;
995+};
996+
997+FIXTURE_VARIANT(iommufd_mock_domain)
998+{
999+ unsigned int mock_domains;
1000+ bool hugepages;
1001+};
1002+
1003+FIXTURE_SETUP(iommufd_mock_domain)
1004+{
1005+ unsigned int i;
1006+
1007+ self->fd = open("/dev/iommu", O_RDWR);
1008+ ASSERT_NE(-1, self->fd);
1009+ test_ioctl_ioas_alloc(&self->ioas_id);
1010+
1011+ ASSERT_GE(ARRAY_SIZE(self->domain_ids), variant->mock_domains);
1012+
1013+ for (i = 0; i != variant->mock_domains; i++)
1014+ test_cmd_mock_domain(self->ioas_id, NULL, &self->domain_ids[i]);
1015+ self->domain_id = self->domain_ids[0];
1016+
1017+ self->mmap_flags = MAP_SHARED | MAP_ANONYMOUS;
1018+ self->mmap_buf_size = PAGE_SIZE * 8;
1019+ if (variant->hugepages) {
1020+ /*
1021+ * MAP_POPULATE will cause the kernel to fail mmap if THPs are
1022+ * not available.
1023+ */
1024+ self->mmap_flags |= MAP_HUGETLB | MAP_POPULATE;
1025+ self->mmap_buf_size = HUGEPAGE_SIZE * 2;
1026+ }
1027+}
1028+
1029+FIXTURE_TEARDOWN(iommufd_mock_domain)
1030+{
1031+ teardown_iommufd(self->fd, _metadata);
1032+}
1033+
1034+FIXTURE_VARIANT_ADD(iommufd_mock_domain, one_domain)
1035+{
1036+ .mock_domains = 1,
1037+ .hugepages = false,
1038+};
1039+
1040+FIXTURE_VARIANT_ADD(iommufd_mock_domain, two_domains)
1041+{
1042+ .mock_domains = 2,
1043+ .hugepages = false,
1044+};
1045+
1046+FIXTURE_VARIANT_ADD(iommufd_mock_domain, one_domain_hugepage)
1047+{
1048+ .mock_domains = 1,
1049+ .hugepages = true,
1050+};
1051+
1052+FIXTURE_VARIANT_ADD(iommufd_mock_domain, two_domains_hugepage)
1053+{
1054+ .mock_domains = 2,
1055+ .hugepages = true,
1056+};
1057+
1058+/* Have the kernel check that the user pages made it to the iommu_domain */
1059+#define check_mock_iova(_ptr, _iova, _length) \
1060+ ({ \
1061+ struct iommu_test_cmd check_map_cmd = { \
1062+ .size = sizeof(check_map_cmd), \
1063+ .op = IOMMU_TEST_OP_MD_CHECK_MAP, \
1064+ .id = self->domain_id, \
1065+ .check_map = { .iova = _iova, \
1066+ .length = _length, \
1067+ .uptr = (uintptr_t)(_ptr) }, \
1068+ }; \
1069+ ASSERT_EQ(0, \
1070+ ioctl(self->fd, \
1071+ _IOMMU_TEST_CMD(IOMMU_TEST_OP_MD_CHECK_MAP), \
1072+ &check_map_cmd)); \
1073+ if (self->domain_ids[1]) { \
1074+ check_map_cmd.id = self->domain_ids[1]; \
1075+ ASSERT_EQ(0, \
1076+ ioctl(self->fd, \
1077+ _IOMMU_TEST_CMD( \
1078+ IOMMU_TEST_OP_MD_CHECK_MAP), \
1079+ &check_map_cmd)); \
1080+ } \
1081+ })
1082+
1083+TEST_F(iommufd_mock_domain, basic)
1084+{
1085+ size_t buf_size = self->mmap_buf_size;
1086+ uint8_t *buf;
1087+ __u64 iova;
1088+
1089+ /* Simple one page map */
1090+ test_ioctl_ioas_map(buffer, PAGE_SIZE, &iova);
1091+ check_mock_iova(buffer, iova, PAGE_SIZE);
1092+
1093+ buf = mmap(0, buf_size, PROT_READ | PROT_WRITE, self->mmap_flags, -1,
1094+ 0);
1095+ ASSERT_NE(MAP_FAILED, buf);
1096+
1097+ /* EFAULT half way through mapping */
1098+ ASSERT_EQ(0, munmap(buf + buf_size / 2, buf_size / 2));
1099+ test_err_ioctl_ioas_map(EFAULT, buf, buf_size, &iova);
1100+
1101+ /* EFAULT on first page */
1102+ ASSERT_EQ(0, munmap(buf, buf_size / 2));
1103+ test_err_ioctl_ioas_map(EFAULT, buf, buf_size, &iova);
1104+}
1105+
1106+TEST_F(iommufd_mock_domain, ro_unshare)
1107+{
1108+ uint8_t *buf;
1109+ __u64 iova;
1110+ int fd;
1111+
1112+ fd = open("/proc/self/exe", O_RDONLY);
1113+ ASSERT_NE(-1, fd);
1114+
1115+ buf = mmap(0, PAGE_SIZE, PROT_READ | PROT_WRITE, MAP_PRIVATE, fd, 0);
1116+ ASSERT_NE(MAP_FAILED, buf);
1117+ close(fd);
1118+
1119+ /*
1120+ * There have been lots of changes to the "unshare" mechanism in
1121+ * get_user_pages(), make sure it works right. The write to the page
1122+ * after we map it for reading should not change the assigned PFN.
1123+ */
1124+ ASSERT_EQ(0,
1125+ _test_ioctl_ioas_map(self->fd, self->ioas_id, buf, PAGE_SIZE,
1126+ &iova, IOMMU_IOAS_MAP_READABLE));
1127+ check_mock_iova(buf, iova, PAGE_SIZE);
1128+ memset(buf, 1, PAGE_SIZE);
1129+ check_mock_iova(buf, iova, PAGE_SIZE);
1130+ ASSERT_EQ(0, munmap(buf, PAGE_SIZE));
1131+}
1132+
1133+TEST_F(iommufd_mock_domain, all_aligns)
1134+{
1135+ size_t test_step = variant->hugepages ? (self->mmap_buf_size / 16) :
1136+ MOCK_PAGE_SIZE;
1137+ size_t buf_size = self->mmap_buf_size;
1138+ unsigned int start;
1139+ unsigned int end;
1140+ uint8_t *buf;
1141+
1142+ buf = mmap(0, buf_size, PROT_READ | PROT_WRITE, self->mmap_flags, -1,
1143+ 0);
1144+ ASSERT_NE(MAP_FAILED, buf);
1145+ check_refs(buf, buf_size, 0);
1146+
1147+ /*
1148+ * Map every combination of page size and alignment within a big region,
1149+ * less for hugepage case as it takes so long to finish.
1150+ */
1151+ for (start = 0; start < buf_size; start += test_step) {
1152+ if (variant->hugepages)
1153+ end = buf_size;
1154+ else
1155+ end = start + MOCK_PAGE_SIZE;
1156+ for (; end < buf_size; end += MOCK_PAGE_SIZE) {
1157+ size_t length = end - start;
1158+ __u64 iova;
1159+
1160+ test_ioctl_ioas_map(buf + start, length, &iova);
1161+ check_mock_iova(buf + start, iova, length);
1162+ check_refs(buf + start / PAGE_SIZE * PAGE_SIZE,
1163+ end / PAGE_SIZE * PAGE_SIZE -
1164+ start / PAGE_SIZE * PAGE_SIZE,
1165+ 1);
1166+
1167+ test_ioctl_ioas_unmap(iova, length);
1168+ }
1169+ }
1170+ check_refs(buf, buf_size, 0);
1171+ ASSERT_EQ(0, munmap(buf, buf_size));
1172+}
1173+
1174+TEST_F(iommufd_mock_domain, all_aligns_copy)
1175+{
1176+ size_t test_step = variant->hugepages ? self->mmap_buf_size / 16 :
1177+ MOCK_PAGE_SIZE;
1178+ size_t buf_size = self->mmap_buf_size;
1179+ unsigned int start;
1180+ unsigned int end;
1181+ uint8_t *buf;
1182+
1183+ buf = mmap(0, buf_size, PROT_READ | PROT_WRITE, self->mmap_flags, -1,
1184+ 0);
1185+ ASSERT_NE(MAP_FAILED, buf);
1186+ check_refs(buf, buf_size, 0);
1187+
1188+ /*
1189+ * Map every combination of page size and alignment within a big region,
1190+ * less for hugepage case as it takes so long to finish.
1191+ */
1192+ for (start = 0; start < buf_size; start += test_step) {
1193+ if (variant->hugepages)
1194+ end = buf_size;
1195+ else
1196+ end = start + MOCK_PAGE_SIZE;
1197+ for (; end < buf_size; end += MOCK_PAGE_SIZE) {
1198+ size_t length = end - start;
1199+ unsigned int old_id;
1200+ uint32_t mock_device_id;
1201+ __u64 iova;
1202+
1203+ test_ioctl_ioas_map(buf + start, length, &iova);
1204+
1205+ /* Add and destroy a domain while the area exists */
1206+ old_id = self->domain_ids[1];
1207+ test_cmd_mock_domain(self->ioas_id, &mock_device_id,
1208+ &self->domain_ids[1]);
1209+
1210+ check_mock_iova(buf + start, iova, length);
1211+ check_refs(buf + start / PAGE_SIZE * PAGE_SIZE,
1212+ end / PAGE_SIZE * PAGE_SIZE -
1213+ start / PAGE_SIZE * PAGE_SIZE,
1214+ 1);
1215+
1216+ test_ioctl_destroy(mock_device_id);
1217+ test_ioctl_destroy(self->domain_ids[1]);
1218+ self->domain_ids[1] = old_id;
1219+
1220+ test_ioctl_ioas_unmap(iova, length);
1221+ }
1222+ }
1223+ check_refs(buf, buf_size, 0);
1224+ ASSERT_EQ(0, munmap(buf, buf_size));
1225+}
1226+
1227+TEST_F(iommufd_mock_domain, user_copy)
1228+{
1229+ struct iommu_test_cmd access_cmd = {
1230+ .size = sizeof(access_cmd),
1231+ .op = IOMMU_TEST_OP_ACCESS_PAGES,
1232+ .access_pages = { .length = BUFFER_SIZE,
1233+ .uptr = (uintptr_t)buffer },
1234+ };
1235+ struct iommu_ioas_copy copy_cmd = {
1236+ .size = sizeof(copy_cmd),
1237+ .flags = IOMMU_IOAS_MAP_FIXED_IOVA,
1238+ .dst_ioas_id = self->ioas_id,
1239+ .dst_iova = MOCK_APERTURE_START,
1240+ .length = BUFFER_SIZE,
1241+ };
1242+ unsigned int ioas_id;
1243+
1244+ /* Pin the pages in an IOAS with no domains then copy to an IOAS with domains */
1245+ test_ioctl_ioas_alloc(&ioas_id);
1246+ test_ioctl_ioas_map_id(ioas_id, buffer, BUFFER_SIZE,
1247+ &copy_cmd.src_iova);
1248+
1249+ test_cmd_create_access(ioas_id, &access_cmd.id,
1250+ MOCK_FLAGS_ACCESS_CREATE_NEEDS_PIN_PAGES);
1251+
1252+ access_cmd.access_pages.iova = copy_cmd.src_iova;
1253+ ASSERT_EQ(0,
1254+ ioctl(self->fd, _IOMMU_TEST_CMD(IOMMU_TEST_OP_ACCESS_PAGES),
1255+ &access_cmd));
1256+ copy_cmd.src_ioas_id = ioas_id;
1257+ ASSERT_EQ(0, ioctl(self->fd, IOMMU_IOAS_COPY, &copy_cmd));
1258+ check_mock_iova(buffer, MOCK_APERTURE_START, BUFFER_SIZE);
1259+
1260+ test_cmd_destroy_access_pages(
1261+ access_cmd.id, access_cmd.access_pages.out_access_pages_id);
1262+ test_cmd_destroy_access(access_cmd.id) test_ioctl_destroy(ioas_id);
1263+
1264+ test_ioctl_destroy(ioas_id);
1265+}
1266+
1267+/* VFIO compatibility IOCTLs */
1268+
1269+TEST_F(iommufd, simple_ioctls)
1270+{
1271+ ASSERT_EQ(VFIO_API_VERSION, ioctl(self->fd, VFIO_GET_API_VERSION));
1272+ ASSERT_EQ(1, ioctl(self->fd, VFIO_CHECK_EXTENSION, VFIO_TYPE1v2_IOMMU));
1273+}
1274+
1275+TEST_F(iommufd, unmap_cmd)
1276+{
1277+ struct vfio_iommu_type1_dma_unmap unmap_cmd = {
1278+ .iova = MOCK_APERTURE_START,
1279+ .size = PAGE_SIZE,
1280+ };
1281+
1282+ unmap_cmd.argsz = 1;
1283+ EXPECT_ERRNO(EINVAL, ioctl(self->fd, VFIO_IOMMU_UNMAP_DMA, &unmap_cmd));
1284+
1285+ unmap_cmd.argsz = sizeof(unmap_cmd);
1286+ unmap_cmd.flags = 1 << 31;
1287+ EXPECT_ERRNO(EINVAL, ioctl(self->fd, VFIO_IOMMU_UNMAP_DMA, &unmap_cmd));
1288+
1289+ unmap_cmd.flags = 0;
1290+ EXPECT_ERRNO(ENODEV, ioctl(self->fd, VFIO_IOMMU_UNMAP_DMA, &unmap_cmd));
1291+}
1292+
1293+TEST_F(iommufd, map_cmd)
1294+{
1295+ struct vfio_iommu_type1_dma_map map_cmd = {
1296+ .iova = MOCK_APERTURE_START,
1297+ .size = PAGE_SIZE,
1298+ .vaddr = (__u64)buffer,
1299+ };
1300+
1301+ map_cmd.argsz = 1;
1302+ EXPECT_ERRNO(EINVAL, ioctl(self->fd, VFIO_IOMMU_MAP_DMA, &map_cmd));
1303+
1304+ map_cmd.argsz = sizeof(map_cmd);
1305+ map_cmd.flags = 1 << 31;
1306+ EXPECT_ERRNO(EINVAL, ioctl(self->fd, VFIO_IOMMU_MAP_DMA, &map_cmd));
1307+
1308+ /* Requires a domain to be attached */
1309+ map_cmd.flags = VFIO_DMA_MAP_FLAG_READ | VFIO_DMA_MAP_FLAG_WRITE;
1310+ EXPECT_ERRNO(ENODEV, ioctl(self->fd, VFIO_IOMMU_MAP_DMA, &map_cmd));
1311+}
1312+
1313+TEST_F(iommufd, info_cmd)
1314+{
1315+ struct vfio_iommu_type1_info info_cmd = {};
1316+
1317+ /* Invalid argsz */
1318+ info_cmd.argsz = 1;
1319+ EXPECT_ERRNO(EINVAL, ioctl(self->fd, VFIO_IOMMU_GET_INFO, &info_cmd));
1320+
1321+ info_cmd.argsz = sizeof(info_cmd);
1322+ EXPECT_ERRNO(ENODEV, ioctl(self->fd, VFIO_IOMMU_GET_INFO, &info_cmd));
1323+}
1324+
1325+TEST_F(iommufd, set_iommu_cmd)
1326+{
1327+ /* Requires a domain to be attached */
1328+ EXPECT_ERRNO(ENODEV,
1329+ ioctl(self->fd, VFIO_SET_IOMMU, VFIO_TYPE1v2_IOMMU));
1330+ EXPECT_ERRNO(ENODEV, ioctl(self->fd, VFIO_SET_IOMMU, VFIO_TYPE1_IOMMU));
1331+}
1332+
1333+TEST_F(iommufd, vfio_ioas)
1334+{
1335+ struct iommu_vfio_ioas vfio_ioas_cmd = {
1336+ .size = sizeof(vfio_ioas_cmd),
1337+ .op = IOMMU_VFIO_IOAS_GET,
1338+ };
1339+ __u32 ioas_id;
1340+
1341+ /* ENODEV if there is no compat ioas */
1342+ EXPECT_ERRNO(ENODEV, ioctl(self->fd, IOMMU_VFIO_IOAS, &vfio_ioas_cmd));
1343+
1344+ /* Invalid id for set */
1345+ vfio_ioas_cmd.op = IOMMU_VFIO_IOAS_SET;
1346+ EXPECT_ERRNO(ENOENT, ioctl(self->fd, IOMMU_VFIO_IOAS, &vfio_ioas_cmd));
1347+
1348+ /* Valid id for set*/
1349+ test_ioctl_ioas_alloc(&ioas_id);
1350+ vfio_ioas_cmd.ioas_id = ioas_id;
1351+ ASSERT_EQ(0, ioctl(self->fd, IOMMU_VFIO_IOAS, &vfio_ioas_cmd));
1352+
1353+ /* Same id comes back from get */
1354+ vfio_ioas_cmd.op = IOMMU_VFIO_IOAS_GET;
1355+ ASSERT_EQ(0, ioctl(self->fd, IOMMU_VFIO_IOAS, &vfio_ioas_cmd));
1356+ ASSERT_EQ(ioas_id, vfio_ioas_cmd.ioas_id);
1357+
1358+ /* Clear works */
1359+ vfio_ioas_cmd.op = IOMMU_VFIO_IOAS_CLEAR;
1360+ ASSERT_EQ(0, ioctl(self->fd, IOMMU_VFIO_IOAS, &vfio_ioas_cmd));
1361+ vfio_ioas_cmd.op = IOMMU_VFIO_IOAS_GET;
1362+ EXPECT_ERRNO(ENODEV, ioctl(self->fd, IOMMU_VFIO_IOAS, &vfio_ioas_cmd));
1363+}
1364+
1365+FIXTURE(vfio_compat_mock_domain)
1366+{
1367+ int fd;
1368+ uint32_t ioas_id;
1369+};
1370+
1371+FIXTURE_VARIANT(vfio_compat_mock_domain)
1372+{
1373+ unsigned int version;
1374+};
1375+
1376+FIXTURE_SETUP(vfio_compat_mock_domain)
1377+{
1378+ struct iommu_vfio_ioas vfio_ioas_cmd = {
1379+ .size = sizeof(vfio_ioas_cmd),
1380+ .op = IOMMU_VFIO_IOAS_SET,
1381+ };
1382+
1383+ self->fd = open("/dev/iommu", O_RDWR);
1384+ ASSERT_NE(-1, self->fd);
1385+
1386+ /* Create what VFIO would consider a group */
1387+ test_ioctl_ioas_alloc(&self->ioas_id);
1388+ test_cmd_mock_domain(self->ioas_id, NULL, NULL);
1389+
1390+ /* Attach it to the vfio compat */
1391+ vfio_ioas_cmd.ioas_id = self->ioas_id;
1392+ ASSERT_EQ(0, ioctl(self->fd, IOMMU_VFIO_IOAS, &vfio_ioas_cmd));
1393+ ASSERT_EQ(0, ioctl(self->fd, VFIO_SET_IOMMU, variant->version));
1394+}
1395+
1396+FIXTURE_TEARDOWN(vfio_compat_mock_domain)
1397+{
1398+ teardown_iommufd(self->fd, _metadata);
1399+}
1400+
1401+FIXTURE_VARIANT_ADD(vfio_compat_mock_domain, Ver1v2)
1402+{
1403+ .version = VFIO_TYPE1v2_IOMMU,
1404+};
1405+
1406+FIXTURE_VARIANT_ADD(vfio_compat_mock_domain, Ver1v0)
1407+{
1408+ .version = VFIO_TYPE1_IOMMU,
1409+};
1410+
1411+TEST_F(vfio_compat_mock_domain, simple_close)
1412+{
1413+}
1414+
1415+TEST_F(vfio_compat_mock_domain, option_huge_pages)
1416+{
1417+ struct iommu_option cmd = {
1418+ .size = sizeof(cmd),
1419+ .option_id = IOMMU_OPTION_HUGE_PAGES,
1420+ .op = IOMMU_OPTION_OP_GET,
1421+ .val64 = 3,
1422+ .object_id = self->ioas_id,
1423+ };
1424+
1425+ ASSERT_EQ(0, ioctl(self->fd, IOMMU_OPTION, &cmd));
1426+ if (variant->version == VFIO_TYPE1_IOMMU) {
1427+ ASSERT_EQ(0, cmd.val64);
1428+ } else {
1429+ ASSERT_EQ(1, cmd.val64);
1430+ }
1431+}
1432+
1433+/*
1434+ * Execute an ioctl command stored in buffer and check that the result does not
1435+ * overflow memory.
1436+ */
1437+static bool is_filled(const void *buf, uint8_t c, size_t len)
1438+{
1439+ const uint8_t *cbuf = buf;
1440+
1441+ for (; len; cbuf++, len--)
1442+ if (*cbuf != c)
1443+ return false;
1444+ return true;
1445+}
1446+
1447+#define ioctl_check_buf(fd, cmd) \
1448+ ({ \
1449+ size_t _cmd_len = *(__u32 *)buffer; \
1450+ \
1451+ memset(buffer + _cmd_len, 0xAA, BUFFER_SIZE - _cmd_len); \
1452+ ASSERT_EQ(0, ioctl(fd, cmd, buffer)); \
1453+ ASSERT_EQ(true, is_filled(buffer + _cmd_len, 0xAA, \
1454+ BUFFER_SIZE - _cmd_len)); \
1455+ })
1456+
1457+static void check_vfio_info_cap_chain(struct __test_metadata *_metadata,
1458+ struct vfio_iommu_type1_info *info_cmd)
1459+{
1460+ const struct vfio_info_cap_header *cap;
1461+
1462+ ASSERT_GE(info_cmd->argsz, info_cmd->cap_offset + sizeof(*cap));
1463+ cap = buffer + info_cmd->cap_offset;
1464+ while (true) {
1465+ size_t cap_size;
1466+
1467+ if (cap->next)
1468+ cap_size = (buffer + cap->next) - (void *)cap;
1469+ else
1470+ cap_size = (buffer + info_cmd->argsz) - (void *)cap;
1471+
1472+ switch (cap->id) {
1473+ case VFIO_IOMMU_TYPE1_INFO_CAP_IOVA_RANGE: {
1474+ struct vfio_iommu_type1_info_cap_iova_range *data =
1475+ (void *)cap;
1476+
1477+ ASSERT_EQ(1, data->header.version);
1478+ ASSERT_EQ(1, data->nr_iovas);
1479+ EXPECT_EQ(MOCK_APERTURE_START,
1480+ data->iova_ranges[0].start);
1481+ EXPECT_EQ(MOCK_APERTURE_LAST, data->iova_ranges[0].end);
1482+ break;
1483+ }
1484+ case VFIO_IOMMU_TYPE1_INFO_DMA_AVAIL: {
1485+ struct vfio_iommu_type1_info_dma_avail *data =
1486+ (void *)cap;
1487+
1488+ ASSERT_EQ(1, data->header.version);
1489+ ASSERT_EQ(sizeof(*data), cap_size);
1490+ break;
1491+ }
1492+ default:
1493+ ASSERT_EQ(false, true);
1494+ break;
1495+ }
1496+ if (!cap->next)
1497+ break;
1498+
1499+ ASSERT_GE(info_cmd->argsz, cap->next + sizeof(*cap));
1500+ ASSERT_GE(buffer + cap->next, (void *)cap);
1501+ cap = buffer + cap->next;
1502+ }
1503+}
1504+
1505+TEST_F(vfio_compat_mock_domain, get_info)
1506+{
1507+ struct vfio_iommu_type1_info *info_cmd = buffer;
1508+ unsigned int i;
1509+ size_t caplen;
1510+
1511+ /* Pre-cap ABI */
1512+ *info_cmd = (struct vfio_iommu_type1_info){
1513+ .argsz = offsetof(struct vfio_iommu_type1_info, cap_offset),
1514+ };
1515+ ioctl_check_buf(self->fd, VFIO_IOMMU_GET_INFO);
1516+ ASSERT_NE(0, info_cmd->iova_pgsizes);
1517+ ASSERT_EQ(VFIO_IOMMU_INFO_PGSIZES | VFIO_IOMMU_INFO_CAPS,
1518+ info_cmd->flags);
1519+
1520+ /* Read the cap chain size */
1521+ *info_cmd = (struct vfio_iommu_type1_info){
1522+ .argsz = sizeof(*info_cmd),
1523+ };
1524+ ioctl_check_buf(self->fd, VFIO_IOMMU_GET_INFO);
1525+ ASSERT_NE(0, info_cmd->iova_pgsizes);
1526+ ASSERT_EQ(VFIO_IOMMU_INFO_PGSIZES | VFIO_IOMMU_INFO_CAPS,
1527+ info_cmd->flags);
1528+ ASSERT_EQ(0, info_cmd->cap_offset);
1529+ ASSERT_LT(sizeof(*info_cmd), info_cmd->argsz);
1530+
1531+ /* Read the caps, kernel should never create a corrupted caps */
1532+ caplen = info_cmd->argsz;
1533+ for (i = sizeof(*info_cmd); i < caplen; i++) {
1534+ *info_cmd = (struct vfio_iommu_type1_info){
1535+ .argsz = i,
1536+ };
1537+ ioctl_check_buf(self->fd, VFIO_IOMMU_GET_INFO);
1538+ ASSERT_EQ(VFIO_IOMMU_INFO_PGSIZES | VFIO_IOMMU_INFO_CAPS,
1539+ info_cmd->flags);
1540+ if (!info_cmd->cap_offset)
1541+ continue;
1542+ check_vfio_info_cap_chain(_metadata, info_cmd);
1543+ }
1544+}
1545+
1546+static void shuffle_array(unsigned long *array, size_t nelms)
1547+{
1548+ unsigned int i;
1549+
1550+ /* Shuffle */
1551+ for (i = 0; i != nelms; i++) {
1552+ unsigned long tmp = array[i];
1553+ unsigned int other = rand() % (nelms - i);
1554+
1555+ array[i] = array[other];
1556+ array[other] = tmp;
1557+ }
1558+}
1559+
1560+TEST_F(vfio_compat_mock_domain, map)
1561+{
1562+ struct vfio_iommu_type1_dma_map map_cmd = {
1563+ .argsz = sizeof(map_cmd),
1564+ .flags = VFIO_DMA_MAP_FLAG_READ | VFIO_DMA_MAP_FLAG_WRITE,
1565+ .vaddr = (uintptr_t)buffer,
1566+ .size = BUFFER_SIZE,
1567+ .iova = MOCK_APERTURE_START,
1568+ };
1569+ struct vfio_iommu_type1_dma_unmap unmap_cmd = {
1570+ .argsz = sizeof(unmap_cmd),
1571+ .size = BUFFER_SIZE,
1572+ .iova = MOCK_APERTURE_START,
1573+ };
1574+ unsigned long pages_iova[BUFFER_SIZE / PAGE_SIZE];
1575+ unsigned int i;
1576+
1577+ /* Simple map/unmap */
1578+ ASSERT_EQ(0, ioctl(self->fd, VFIO_IOMMU_MAP_DMA, &map_cmd));
1579+ ASSERT_EQ(0, ioctl(self->fd, VFIO_IOMMU_UNMAP_DMA, &unmap_cmd));
1580+ ASSERT_EQ(BUFFER_SIZE, unmap_cmd.size);
1581+
1582+ /* UNMAP_FLAG_ALL requres 0 iova/size */
1583+ ASSERT_EQ(0, ioctl(self->fd, VFIO_IOMMU_MAP_DMA, &map_cmd));
1584+ unmap_cmd.flags = VFIO_DMA_UNMAP_FLAG_ALL;
1585+ EXPECT_ERRNO(EINVAL, ioctl(self->fd, VFIO_IOMMU_UNMAP_DMA, &unmap_cmd));
1586+
1587+ unmap_cmd.iova = 0;
1588+ unmap_cmd.size = 0;
1589+ ASSERT_EQ(0, ioctl(self->fd, VFIO_IOMMU_UNMAP_DMA, &unmap_cmd));
1590+ ASSERT_EQ(BUFFER_SIZE, unmap_cmd.size);
1591+
1592+ /* Small pages */
1593+ for (i = 0; i != ARRAY_SIZE(pages_iova); i++) {
1594+ map_cmd.iova = pages_iova[i] =
1595+ MOCK_APERTURE_START + i * PAGE_SIZE;
1596+ map_cmd.vaddr = (uintptr_t)buffer + i * PAGE_SIZE;
1597+ map_cmd.size = PAGE_SIZE;
1598+ ASSERT_EQ(0, ioctl(self->fd, VFIO_IOMMU_MAP_DMA, &map_cmd));
1599+ }
1600+ shuffle_array(pages_iova, ARRAY_SIZE(pages_iova));
1601+
1602+ unmap_cmd.flags = 0;
1603+ unmap_cmd.size = PAGE_SIZE;
1604+ for (i = 0; i != ARRAY_SIZE(pages_iova); i++) {
1605+ unmap_cmd.iova = pages_iova[i];
1606+ ASSERT_EQ(0, ioctl(self->fd, VFIO_IOMMU_UNMAP_DMA, &unmap_cmd));
1607+ }
1608+}
1609+
1610+TEST_F(vfio_compat_mock_domain, huge_map)
1611+{
1612+ size_t buf_size = HUGEPAGE_SIZE * 2;
1613+ struct vfio_iommu_type1_dma_map map_cmd = {
1614+ .argsz = sizeof(map_cmd),
1615+ .flags = VFIO_DMA_MAP_FLAG_READ | VFIO_DMA_MAP_FLAG_WRITE,
1616+ .size = buf_size,
1617+ .iova = MOCK_APERTURE_START,
1618+ };
1619+ struct vfio_iommu_type1_dma_unmap unmap_cmd = {
1620+ .argsz = sizeof(unmap_cmd),
1621+ };
1622+ unsigned long pages_iova[16];
1623+ unsigned int i;
1624+ void *buf;
1625+
1626+ /* Test huge pages and splitting */
1627+ buf = mmap(0, buf_size, PROT_READ | PROT_WRITE,
1628+ MAP_SHARED | MAP_ANONYMOUS | MAP_HUGETLB | MAP_POPULATE, -1,
1629+ 0);
1630+ ASSERT_NE(MAP_FAILED, buf);
1631+ map_cmd.vaddr = (uintptr_t)buf;
1632+ ASSERT_EQ(0, ioctl(self->fd, VFIO_IOMMU_MAP_DMA, &map_cmd));
1633+
1634+ unmap_cmd.size = buf_size / ARRAY_SIZE(pages_iova);
1635+ for (i = 0; i != ARRAY_SIZE(pages_iova); i++)
1636+ pages_iova[i] = MOCK_APERTURE_START + (i * unmap_cmd.size);
1637+ shuffle_array(pages_iova, ARRAY_SIZE(pages_iova));
1638+
1639+ /* type1 mode can cut up larger mappings, type1v2 always fails */
1640+ for (i = 0; i != ARRAY_SIZE(pages_iova); i++) {
1641+ unmap_cmd.iova = pages_iova[i];
1642+ unmap_cmd.size = buf_size / ARRAY_SIZE(pages_iova);
1643+ if (variant->version == VFIO_TYPE1_IOMMU) {
1644+ ASSERT_EQ(0, ioctl(self->fd, VFIO_IOMMU_UNMAP_DMA,
1645+ &unmap_cmd));
1646+ } else {
1647+ EXPECT_ERRNO(ENOENT,
1648+ ioctl(self->fd, VFIO_IOMMU_UNMAP_DMA,
1649+ &unmap_cmd));
1650+ }
1651+ }
1652+}
1653+
1654+TEST_HARNESS_MAIN
tools/testing/selftests/iommu/iommufd_fail_nth.c+580−0
@@ -0,0 +1,580 @@
1+// SPDX-License-Identifier: GPL-2.0-only
2+/* Copyright (c) 2021-2022, NVIDIA CORPORATION & AFFILIATES
3+ *
4+ * These tests are "kernel integrity" tests. They are looking for kernel
5+ * WARN/OOPS/kasn/etc splats triggered by kernel sanitizers & debugging
6+ * features. It does not attempt to verify that the system calls are doing what
7+ * they are supposed to do.
8+ *
9+ * The basic philosophy is to run a sequence of calls that will succeed and then
10+ * sweep every failure injection point on that call chain to look for
11+ * interesting things in error handling.
12+ *
13+ * This test is best run with:
14+ * echo 1 > /proc/sys/kernel/panic_on_warn
15+ * If something is actually going wrong.
16+ */
17+#include <fcntl.h>
18+#include <dirent.h>
19+
20+#define __EXPORTED_HEADERS__
21+#include <linux/vfio.h>
22+
23+#include "iommufd_utils.h"
24+
25+static bool have_fault_injection;
26+
27+static int writeat(int dfd, const char *fn, const char *val)
28+{
29+ size_t val_len = strlen(val);
30+ ssize_t res;
31+ int fd;
32+
33+ fd = openat(dfd, fn, O_WRONLY);
34+ if (fd == -1)
35+ return -1;
36+ res = write(fd, val, val_len);
37+ assert(res == val_len);
38+ close(fd);
39+ return 0;
40+}
41+
42+static __attribute__((constructor)) void setup_buffer(void)
43+{
44+ BUFFER_SIZE = 2*1024*1024;
45+
46+ buffer = mmap(0, BUFFER_SIZE, PROT_READ | PROT_WRITE,
47+ MAP_SHARED | MAP_ANONYMOUS, -1, 0);
48+}
49+
50+/*
51+ * This sets up fail_injection in a way that is useful for this test.
52+ * It does not attempt to restore things back to how they were.
53+ */
54+static __attribute__((constructor)) void setup_fault_injection(void)
55+{
56+ DIR *debugfs = opendir("/sys/kernel/debug/");
57+ struct dirent *dent;
58+
59+ if (!debugfs)
60+ return;
61+
62+ /* Allow any allocation call to be fault injected */
63+ if (writeat(dirfd(debugfs), "failslab/ignore-gfp-wait", "N"))
64+ return;
65+ writeat(dirfd(debugfs), "fail_page_alloc/ignore-gfp-wait", "N");
66+ writeat(dirfd(debugfs), "fail_page_alloc/ignore-gfp-highmem", "N");
67+
68+ while ((dent = readdir(debugfs))) {
69+ char fn[300];
70+
71+ if (strncmp(dent->d_name, "fail", 4) != 0)
72+ continue;
73+
74+ /* We are looking for kernel splats, quiet down the log */
75+ snprintf(fn, sizeof(fn), "%s/verbose", dent->d_name);
76+ writeat(dirfd(debugfs), fn, "0");
77+ }
78+ closedir(debugfs);
79+ have_fault_injection = true;
80+}
81+
82+struct fail_nth_state {
83+ int proc_fd;
84+ unsigned int iteration;
85+};
86+
87+static void fail_nth_first(struct __test_metadata *_metadata,
88+ struct fail_nth_state *nth_state)
89+{
90+ char buf[300];
91+
92+ snprintf(buf, sizeof(buf), "/proc/self/task/%u/fail-nth", getpid());
93+ nth_state->proc_fd = open(buf, O_RDWR);
94+ ASSERT_NE(-1, nth_state->proc_fd);
95+}
96+
97+static bool fail_nth_next(struct __test_metadata *_metadata,
98+ struct fail_nth_state *nth_state,
99+ int test_result)
100+{
101+ static const char disable_nth[] = "0";
102+ char buf[300];
103+
104+ /*
105+ * This is just an arbitrary limit based on the current kernel
106+ * situation. Changes in the kernel can dramtically change the number of
107+ * required fault injection sites, so if this hits it doesn't
108+ * necessarily mean a test failure, just that the limit has to be made
109+ * bigger.
110+ */
111+ ASSERT_GT(400, nth_state->iteration);
112+ if (nth_state->iteration != 0) {
113+ ssize_t res;
114+ ssize_t res2;
115+
116+ buf[0] = 0;
117+ /*
118+ * Annoyingly disabling the nth can also fail. This means
119+ * the test passed without triggering failure
120+ */
121+ res = pread(nth_state->proc_fd, buf, sizeof(buf), 0);
122+ if (res == -1 && errno == EFAULT) {
123+ buf[0] = '1';
124+ buf[1] = '\n';
125+ res = 2;
126+ }
127+
128+ res2 = pwrite(nth_state->proc_fd, disable_nth,
129+ ARRAY_SIZE(disable_nth) - 1, 0);
130+ if (res2 == -1 && errno == EFAULT) {
131+ res2 = pwrite(nth_state->proc_fd, disable_nth,
132+ ARRAY_SIZE(disable_nth) - 1, 0);
133+ buf[0] = '1';
134+ buf[1] = '\n';
135+ }
136+ ASSERT_EQ(ARRAY_SIZE(disable_nth) - 1, res2);
137+
138+ /* printf(" nth %u result=%d nth=%u\n", nth_state->iteration,
139+ test_result, atoi(buf)); */
140+ fflush(stdout);
141+ ASSERT_LT(1, res);
142+ if (res != 2 || buf[0] != '0' || buf[1] != '\n')
143+ return false;
144+ } else {
145+ /* printf(" nth %u result=%d\n", nth_state->iteration,
146+ test_result); */
147+ }
148+ nth_state->iteration++;
149+ return true;
150+}
151+
152+/*
153+ * This is called during the test to start failure injection. It allows the test
154+ * to do some setup that has already been swept and thus reduce the required
155+ * iterations.
156+ */
157+void __fail_nth_enable(struct __test_metadata *_metadata,
158+ struct fail_nth_state *nth_state)
159+{
160+ char buf[300];
161+ size_t len;
162+
163+ if (!nth_state->iteration)
164+ return;
165+
166+ len = snprintf(buf, sizeof(buf), "%u", nth_state->iteration);
167+ ASSERT_EQ(len, pwrite(nth_state->proc_fd, buf, len, 0));
168+}
169+#define fail_nth_enable() __fail_nth_enable(_metadata, _nth_state)
170+
171+#define TEST_FAIL_NTH(fixture_name, name) \
172+ static int test_nth_##name(struct __test_metadata *_metadata, \
173+ FIXTURE_DATA(fixture_name) *self, \
174+ const FIXTURE_VARIANT(fixture_name) \
175+ *variant, \
176+ struct fail_nth_state *_nth_state); \
177+ TEST_F(fixture_name, name) \
178+ { \
179+ struct fail_nth_state nth_state = {}; \
180+ int test_result = 0; \
181+ \
182+ if (!have_fault_injection) \
183+ SKIP(return, \
184+ "fault injection is not enabled in the kernel"); \
185+ fail_nth_first(_metadata, &nth_state); \
186+ ASSERT_EQ(0, test_nth_##name(_metadata, self, variant, \
187+ &nth_state)); \
188+ while (fail_nth_next(_metadata, &nth_state, test_result)) { \
189+ fixture_name##_teardown(_metadata, self, variant); \
190+ fixture_name##_setup(_metadata, self, variant); \
191+ test_result = test_nth_##name(_metadata, self, \
192+ variant, &nth_state); \
193+ }; \
194+ ASSERT_EQ(0, test_result); \
195+ } \
196+ static int test_nth_##name( \
197+ struct __test_metadata __attribute__((unused)) *_metadata, \
198+ FIXTURE_DATA(fixture_name) __attribute__((unused)) *self, \
199+ const FIXTURE_VARIANT(fixture_name) __attribute__((unused)) \
200+ *variant, \
201+ struct fail_nth_state *_nth_state)
202+
203+FIXTURE(basic_fail_nth)
204+{
205+ int fd;
206+ uint32_t access_id;
207+};
208+
209+FIXTURE_SETUP(basic_fail_nth)
210+{
211+ self->fd = -1;
212+ self->access_id = 0;
213+}
214+
215+FIXTURE_TEARDOWN(basic_fail_nth)
216+{
217+ int rc;
218+
219+ if (self->access_id) {
220+ /* The access FD holds the iommufd open until it closes */
221+ rc = _test_cmd_destroy_access(self->access_id);
222+ assert(rc == 0);
223+ }
224+ teardown_iommufd(self->fd, _metadata);
225+}
226+
227+/* Cover ioas.c */
228+TEST_FAIL_NTH(basic_fail_nth, basic)
229+{
230+ struct iommu_iova_range ranges[10];
231+ uint32_t ioas_id;
232+ __u64 iova;
233+
234+ fail_nth_enable();
235+
236+ self->fd = open("/dev/iommu", O_RDWR);
237+ if (self->fd == -1)
238+ return -1;
239+
240+ if (_test_ioctl_ioas_alloc(self->fd, &ioas_id))
241+ return -1;
242+
243+ {
244+ struct iommu_ioas_iova_ranges ranges_cmd = {
245+ .size = sizeof(ranges_cmd),
246+ .num_iovas = ARRAY_SIZE(ranges),
247+ .ioas_id = ioas_id,
248+ .allowed_iovas = (uintptr_t)ranges,
249+ };
250+ if (ioctl(self->fd, IOMMU_IOAS_IOVA_RANGES, &ranges_cmd))
251+ return -1;
252+ }
253+
254+ {
255+ struct iommu_ioas_allow_iovas allow_cmd = {
256+ .size = sizeof(allow_cmd),
257+ .ioas_id = ioas_id,
258+ .num_iovas = 1,
259+ .allowed_iovas = (uintptr_t)ranges,
260+ };
261+
262+ ranges[0].start = 16*1024;
263+ ranges[0].last = BUFFER_SIZE + 16 * 1024 * 600 - 1;
264+ if (ioctl(self->fd, IOMMU_IOAS_ALLOW_IOVAS, &allow_cmd))
265+ return -1;
266+ }
267+
268+ if (_test_ioctl_ioas_map(self->fd, ioas_id, buffer, BUFFER_SIZE, &iova,
269+ IOMMU_IOAS_MAP_WRITEABLE |
270+ IOMMU_IOAS_MAP_READABLE))
271+ return -1;
272+
273+ {
274+ struct iommu_ioas_copy copy_cmd = {
275+ .size = sizeof(copy_cmd),
276+ .flags = IOMMU_IOAS_MAP_WRITEABLE |
277+ IOMMU_IOAS_MAP_READABLE,
278+ .dst_ioas_id = ioas_id,
279+ .src_ioas_id = ioas_id,
280+ .src_iova = iova,
281+ .length = sizeof(ranges),
282+ };
283+
284+ if (ioctl(self->fd, IOMMU_IOAS_COPY, &copy_cmd))
285+ return -1;
286+ }
287+
288+ if (_test_ioctl_ioas_unmap(self->fd, ioas_id, iova, BUFFER_SIZE,
289+ NULL))
290+ return -1;
291+ /* Failure path of no IOVA to unmap */
292+ _test_ioctl_ioas_unmap(self->fd, ioas_id, iova, BUFFER_SIZE, NULL);
293+ return 0;
294+}
295+
296+/* iopt_area_fill_domains() and iopt_area_fill_domain() */
297+TEST_FAIL_NTH(basic_fail_nth, map_domain)
298+{
299+ uint32_t ioas_id;
300+ __u32 device_id;
301+ __u32 hwpt_id;
302+ __u64 iova;
303+
304+ self->fd = open("/dev/iommu", O_RDWR);
305+ if (self->fd == -1)
306+ return -1;
307+
308+ if (_test_ioctl_ioas_alloc(self->fd, &ioas_id))
309+ return -1;
310+
311+ if (_test_ioctl_set_temp_memory_limit(self->fd, 32))
312+ return -1;
313+
314+ fail_nth_enable();
315+
316+ if (_test_cmd_mock_domain(self->fd, ioas_id, &device_id, &hwpt_id))
317+ return -1;
318+
319+ if (_test_ioctl_ioas_map(self->fd, ioas_id, buffer, 262144, &iova,
320+ IOMMU_IOAS_MAP_WRITEABLE |
321+ IOMMU_IOAS_MAP_READABLE))
322+ return -1;
323+
324+ if (_test_ioctl_destroy(self->fd, device_id))
325+ return -1;
326+ if (_test_ioctl_destroy(self->fd, hwpt_id))
327+ return -1;
328+
329+ if (_test_cmd_mock_domain(self->fd, ioas_id, &device_id, &hwpt_id))
330+ return -1;
331+ return 0;
332+}
333+
334+TEST_FAIL_NTH(basic_fail_nth, map_two_domains)
335+{
336+ uint32_t ioas_id;
337+ __u32 device_id2;
338+ __u32 device_id;
339+ __u32 hwpt_id2;
340+ __u32 hwpt_id;
341+ __u64 iova;
342+
343+ self->fd = open("/dev/iommu", O_RDWR);
344+ if (self->fd == -1)
345+ return -1;
346+
347+ if (_test_ioctl_ioas_alloc(self->fd, &ioas_id))
348+ return -1;
349+
350+ if (_test_ioctl_set_temp_memory_limit(self->fd, 32))
351+ return -1;
352+
353+ if (_test_cmd_mock_domain(self->fd, ioas_id, &device_id, &hwpt_id))
354+ return -1;
355+
356+ fail_nth_enable();
357+
358+ if (_test_cmd_mock_domain(self->fd, ioas_id, &device_id2, &hwpt_id2))
359+ return -1;
360+
361+ if (_test_ioctl_ioas_map(self->fd, ioas_id, buffer, 262144, &iova,
362+ IOMMU_IOAS_MAP_WRITEABLE |
363+ IOMMU_IOAS_MAP_READABLE))
364+ return -1;
365+
366+ if (_test_ioctl_destroy(self->fd, device_id))
367+ return -1;
368+ if (_test_ioctl_destroy(self->fd, hwpt_id))
369+ return -1;
370+
371+ if (_test_ioctl_destroy(self->fd, device_id2))
372+ return -1;
373+ if (_test_ioctl_destroy(self->fd, hwpt_id2))
374+ return -1;
375+
376+ if (_test_cmd_mock_domain(self->fd, ioas_id, &device_id, &hwpt_id))
377+ return -1;
378+ if (_test_cmd_mock_domain(self->fd, ioas_id, &device_id2, &hwpt_id2))
379+ return -1;
380+ return 0;
381+}
382+
383+TEST_FAIL_NTH(basic_fail_nth, access_rw)
384+{
385+ uint64_t tmp_big[4096];
386+ uint32_t ioas_id;
387+ uint16_t tmp[32];
388+ __u64 iova;
389+
390+ self->fd = open("/dev/iommu", O_RDWR);
391+ if (self->fd == -1)
392+ return -1;
393+
394+ if (_test_ioctl_ioas_alloc(self->fd, &ioas_id))
395+ return -1;
396+
397+ if (_test_ioctl_set_temp_memory_limit(self->fd, 32))
398+ return -1;
399+
400+ if (_test_ioctl_ioas_map(self->fd, ioas_id, buffer, 262144, &iova,
401+ IOMMU_IOAS_MAP_WRITEABLE |
402+ IOMMU_IOAS_MAP_READABLE))
403+ return -1;
404+
405+ fail_nth_enable();
406+
407+ if (_test_cmd_create_access(self->fd, ioas_id, &self->access_id, 0))
408+ return -1;
409+
410+ {
411+ struct iommu_test_cmd access_cmd = {
412+ .size = sizeof(access_cmd),
413+ .op = IOMMU_TEST_OP_ACCESS_RW,
414+ .id = self->access_id,
415+ .access_rw = { .iova = iova,
416+ .length = sizeof(tmp),
417+ .uptr = (uintptr_t)tmp },
418+ };
419+
420+ // READ
421+ if (ioctl(self->fd, _IOMMU_TEST_CMD(IOMMU_TEST_OP_ACCESS_RW),
422+ &access_cmd))
423+ return -1;
424+
425+ access_cmd.access_rw.flags = MOCK_ACCESS_RW_WRITE;
426+ if (ioctl(self->fd, _IOMMU_TEST_CMD(IOMMU_TEST_OP_ACCESS_RW),
427+ &access_cmd))
428+ return -1;
429+
430+ access_cmd.access_rw.flags = MOCK_ACCESS_RW_SLOW_PATH;
431+ if (ioctl(self->fd, _IOMMU_TEST_CMD(IOMMU_TEST_OP_ACCESS_RW),
432+ &access_cmd))
433+ return -1;
434+ access_cmd.access_rw.flags = MOCK_ACCESS_RW_SLOW_PATH |
435+ MOCK_ACCESS_RW_WRITE;
436+ if (ioctl(self->fd, _IOMMU_TEST_CMD(IOMMU_TEST_OP_ACCESS_RW),
437+ &access_cmd))
438+ return -1;
439+ }
440+
441+ {
442+ struct iommu_test_cmd access_cmd = {
443+ .size = sizeof(access_cmd),
444+ .op = IOMMU_TEST_OP_ACCESS_RW,
445+ .id = self->access_id,
446+ .access_rw = { .iova = iova,
447+ .flags = MOCK_ACCESS_RW_SLOW_PATH,
448+ .length = sizeof(tmp_big),
449+ .uptr = (uintptr_t)tmp_big },
450+ };
451+
452+ if (ioctl(self->fd, _IOMMU_TEST_CMD(IOMMU_TEST_OP_ACCESS_RW),
453+ &access_cmd))
454+ return -1;
455+ }
456+ if (_test_cmd_destroy_access(self->access_id))
457+ return -1;
458+ self->access_id = 0;
459+ return 0;
460+}
461+
462+/* pages.c access functions */
463+TEST_FAIL_NTH(basic_fail_nth, access_pin)
464+{
465+ uint32_t access_pages_id;
466+ uint32_t ioas_id;
467+ __u64 iova;
468+
469+ self->fd = open("/dev/iommu", O_RDWR);
470+ if (self->fd == -1)
471+ return -1;
472+
473+ if (_test_ioctl_ioas_alloc(self->fd, &ioas_id))
474+ return -1;
475+
476+ if (_test_ioctl_set_temp_memory_limit(self->fd, 32))
477+ return -1;
478+
479+ if (_test_ioctl_ioas_map(self->fd, ioas_id, buffer, BUFFER_SIZE, &iova,
480+ IOMMU_IOAS_MAP_WRITEABLE |
481+ IOMMU_IOAS_MAP_READABLE))
482+ return -1;
483+
484+ if (_test_cmd_create_access(self->fd, ioas_id, &self->access_id,
485+ MOCK_FLAGS_ACCESS_CREATE_NEEDS_PIN_PAGES))
486+ return -1;
487+
488+ fail_nth_enable();
489+
490+ {
491+ struct iommu_test_cmd access_cmd = {
492+ .size = sizeof(access_cmd),
493+ .op = IOMMU_TEST_OP_ACCESS_PAGES,
494+ .id = self->access_id,
495+ .access_pages = { .iova = iova,
496+ .length = BUFFER_SIZE,
497+ .uptr = (uintptr_t)buffer },
498+ };
499+
500+ if (ioctl(self->fd, _IOMMU_TEST_CMD(IOMMU_TEST_OP_ACCESS_RW),
501+ &access_cmd))
502+ return -1;
503+ access_pages_id = access_cmd.access_pages.out_access_pages_id;
504+ }
505+
506+ if (_test_cmd_destroy_access_pages(self->fd, self->access_id,
507+ access_pages_id))
508+ return -1;
509+
510+ if (_test_cmd_destroy_access(self->access_id))
511+ return -1;
512+ self->access_id = 0;
513+ return 0;
514+}
515+
516+/* iopt_pages_fill_xarray() */
517+TEST_FAIL_NTH(basic_fail_nth, access_pin_domain)
518+{
519+ uint32_t access_pages_id;
520+ uint32_t ioas_id;
521+ __u32 device_id;
522+ __u32 hwpt_id;
523+ __u64 iova;
524+
525+ self->fd = open("/dev/iommu", O_RDWR);
526+ if (self->fd == -1)
527+ return -1;
528+
529+ if (_test_ioctl_ioas_alloc(self->fd, &ioas_id))
530+ return -1;
531+
532+ if (_test_ioctl_set_temp_memory_limit(self->fd, 32))
533+ return -1;
534+
535+ if (_test_cmd_mock_domain(self->fd, ioas_id, &device_id, &hwpt_id))
536+ return -1;
537+
538+ if (_test_ioctl_ioas_map(self->fd, ioas_id, buffer, BUFFER_SIZE, &iova,
539+ IOMMU_IOAS_MAP_WRITEABLE |
540+ IOMMU_IOAS_MAP_READABLE))
541+ return -1;
542+
543+ if (_test_cmd_create_access(self->fd, ioas_id, &self->access_id,
544+ MOCK_FLAGS_ACCESS_CREATE_NEEDS_PIN_PAGES))
545+ return -1;
546+
547+ fail_nth_enable();
548+
549+ {
550+ struct iommu_test_cmd access_cmd = {
551+ .size = sizeof(access_cmd),
552+ .op = IOMMU_TEST_OP_ACCESS_PAGES,
553+ .id = self->access_id,
554+ .access_pages = { .iova = iova,
555+ .length = BUFFER_SIZE,
556+ .uptr = (uintptr_t)buffer },
557+ };
558+
559+ if (ioctl(self->fd, _IOMMU_TEST_CMD(IOMMU_TEST_OP_ACCESS_RW),
560+ &access_cmd))
561+ return -1;
562+ access_pages_id = access_cmd.access_pages.out_access_pages_id;
563+ }
564+
565+ if (_test_cmd_destroy_access_pages(self->fd, self->access_id,
566+ access_pages_id))
567+ return -1;
568+
569+ if (_test_cmd_destroy_access(self->access_id))
570+ return -1;
571+ self->access_id = 0;
572+
573+ if (_test_ioctl_destroy(self->fd, device_id))
574+ return -1;
575+ if (_test_ioctl_destroy(self->fd, hwpt_id))
576+ return -1;
577+ return 0;
578+}
579+
580+TEST_HARNESS_MAIN
tools/testing/selftests/iommu/iommufd_utils.h+278−0
@@ -0,0 +1,278 @@
1+/* SPDX-License-Identifier: GPL-2.0-only */
2+/* Copyright (c) 2021-2022, NVIDIA CORPORATION & AFFILIATES */
3+#ifndef __SELFTEST_IOMMUFD_UTILS
4+#define __SELFTEST_IOMMUFD_UTILS
5+
6+#include <unistd.h>
7+#include <stddef.h>
8+#include <sys/fcntl.h>
9+#include <sys/ioctl.h>
10+#include <stdint.h>
11+#include <assert.h>
12+
13+#include "../kselftest_harness.h"
14+#include "../../../../drivers/iommu/iommufd/iommufd_test.h"
15+
16+/* Hack to make assertions more readable */
17+#define _IOMMU_TEST_CMD(x) IOMMU_TEST_CMD
18+
19+static void *buffer;
20+static unsigned long BUFFER_SIZE;
21+
22+/*
23+ * Have the kernel check the refcount on pages. I don't know why a freshly
24+ * mmap'd anon non-compound page starts out with a ref of 3
25+ */
26+#define check_refs(_ptr, _length, _refs) \
27+ ({ \
28+ struct iommu_test_cmd test_cmd = { \
29+ .size = sizeof(test_cmd), \
30+ .op = IOMMU_TEST_OP_MD_CHECK_REFS, \
31+ .check_refs = { .length = _length, \
32+ .uptr = (uintptr_t)(_ptr), \
33+ .refs = _refs }, \
34+ }; \
35+ ASSERT_EQ(0, \
36+ ioctl(self->fd, \
37+ _IOMMU_TEST_CMD(IOMMU_TEST_OP_MD_CHECK_REFS), \
38+ &test_cmd)); \
39+ })
40+
41+static int _test_cmd_mock_domain(int fd, unsigned int ioas_id, __u32 *device_id,
42+ __u32 *hwpt_id)
43+{
44+ struct iommu_test_cmd cmd = {
45+ .size = sizeof(cmd),
46+ .op = IOMMU_TEST_OP_MOCK_DOMAIN,
47+ .id = ioas_id,
48+ .mock_domain = {},
49+ };
50+ int ret;
51+
52+ ret = ioctl(fd, IOMMU_TEST_CMD, &cmd);
53+ if (ret)
54+ return ret;
55+ if (device_id)
56+ *device_id = cmd.mock_domain.out_device_id;
57+ assert(cmd.id != 0);
58+ if (hwpt_id)
59+ *hwpt_id = cmd.mock_domain.out_hwpt_id;
60+ return 0;
61+}
62+#define test_cmd_mock_domain(ioas_id, device_id, hwpt_id) \
63+ ASSERT_EQ(0, _test_cmd_mock_domain(self->fd, ioas_id, device_id, \
64+ hwpt_id))
65+#define test_err_mock_domain(_errno, ioas_id, device_id, hwpt_id) \
66+ EXPECT_ERRNO(_errno, _test_cmd_mock_domain(self->fd, ioas_id, \
67+ device_id, hwpt_id))
68+
69+static int _test_cmd_create_access(int fd, unsigned int ioas_id,
70+ __u32 *access_id, unsigned int flags)
71+{
72+ struct iommu_test_cmd cmd = {
73+ .size = sizeof(cmd),
74+ .op = IOMMU_TEST_OP_CREATE_ACCESS,
75+ .id = ioas_id,
76+ .create_access = { .flags = flags },
77+ };
78+ int ret;
79+
80+ ret = ioctl(fd, IOMMU_TEST_CMD, &cmd);
81+ if (ret)
82+ return ret;
83+ *access_id = cmd.create_access.out_access_fd;
84+ return 0;
85+}
86+#define test_cmd_create_access(ioas_id, access_id, flags) \
87+ ASSERT_EQ(0, _test_cmd_create_access(self->fd, ioas_id, access_id, \
88+ flags))
89+
90+static int _test_cmd_destroy_access(unsigned int access_id)
91+{
92+ return close(access_id);
93+}
94+#define test_cmd_destroy_access(access_id) \
95+ ASSERT_EQ(0, _test_cmd_destroy_access(access_id))
96+
97+static int _test_cmd_destroy_access_pages(int fd, unsigned int access_id,
98+ unsigned int access_pages_id)
99+{
100+ struct iommu_test_cmd cmd = {
101+ .size = sizeof(cmd),
102+ .op = IOMMU_TEST_OP_DESTROY_ACCESS_PAGES,
103+ .id = access_id,
104+ .destroy_access_pages = { .access_pages_id = access_pages_id },
105+ };
106+ return ioctl(fd, IOMMU_TEST_CMD, &cmd);
107+}
108+#define test_cmd_destroy_access_pages(access_id, access_pages_id) \
109+ ASSERT_EQ(0, _test_cmd_destroy_access_pages(self->fd, access_id, \
110+ access_pages_id))
111+#define test_err_destroy_access_pages(_errno, access_id, access_pages_id) \
112+ EXPECT_ERRNO(_errno, _test_cmd_destroy_access_pages( \
113+ self->fd, access_id, access_pages_id))
114+
115+static int _test_ioctl_destroy(int fd, unsigned int id)
116+{
117+ struct iommu_destroy cmd = {
118+ .size = sizeof(cmd),
119+ .id = id,
120+ };
121+ return ioctl(fd, IOMMU_DESTROY, &cmd);
122+}
123+#define test_ioctl_destroy(id) ASSERT_EQ(0, _test_ioctl_destroy(self->fd, id))
124+
125+static int _test_ioctl_ioas_alloc(int fd, __u32 *id)
126+{
127+ struct iommu_ioas_alloc cmd = {
128+ .size = sizeof(cmd),
129+ };
130+ int ret;
131+
132+ ret = ioctl(fd, IOMMU_IOAS_ALLOC, &cmd);
133+ if (ret)
134+ return ret;
135+ *id = cmd.out_ioas_id;
136+ return 0;
137+}
138+#define test_ioctl_ioas_alloc(id) \
139+ ({ \
140+ ASSERT_EQ(0, _test_ioctl_ioas_alloc(self->fd, id)); \
141+ ASSERT_NE(0, *(id)); \
142+ })
143+
144+static int _test_ioctl_ioas_map(int fd, unsigned int ioas_id, void *buffer,
145+ size_t length, __u64 *iova, unsigned int flags)
146+{
147+ struct iommu_ioas_map cmd = {
148+ .size = sizeof(cmd),
149+ .flags = flags,
150+ .ioas_id = ioas_id,
151+ .user_va = (uintptr_t)buffer,
152+ .length = length,
153+ };
154+ int ret;
155+
156+ if (flags & IOMMU_IOAS_MAP_FIXED_IOVA)
157+ cmd.iova = *iova;
158+
159+ ret = ioctl(fd, IOMMU_IOAS_MAP, &cmd);
160+ *iova = cmd.iova;
161+ return ret;
162+}
163+#define test_ioctl_ioas_map(buffer, length, iova_p) \
164+ ASSERT_EQ(0, _test_ioctl_ioas_map(self->fd, self->ioas_id, buffer, \
165+ length, iova_p, \
166+ IOMMU_IOAS_MAP_WRITEABLE | \
167+ IOMMU_IOAS_MAP_READABLE))
168+
169+#define test_err_ioctl_ioas_map(_errno, buffer, length, iova_p) \
170+ EXPECT_ERRNO(_errno, \
171+ _test_ioctl_ioas_map(self->fd, self->ioas_id, buffer, \
172+ length, iova_p, \
173+ IOMMU_IOAS_MAP_WRITEABLE | \
174+ IOMMU_IOAS_MAP_READABLE))
175+
176+#define test_ioctl_ioas_map_id(ioas_id, buffer, length, iova_p) \
177+ ASSERT_EQ(0, _test_ioctl_ioas_map(self->fd, ioas_id, buffer, length, \
178+ iova_p, \
179+ IOMMU_IOAS_MAP_WRITEABLE | \
180+ IOMMU_IOAS_MAP_READABLE))
181+
182+#define test_ioctl_ioas_map_fixed(buffer, length, iova) \
183+ ({ \
184+ __u64 __iova = iova; \
185+ ASSERT_EQ(0, _test_ioctl_ioas_map( \
186+ self->fd, self->ioas_id, buffer, length, \
187+ &__iova, \
188+ IOMMU_IOAS_MAP_FIXED_IOVA | \
189+ IOMMU_IOAS_MAP_WRITEABLE | \
190+ IOMMU_IOAS_MAP_READABLE)); \
191+ })
192+
193+#define test_err_ioctl_ioas_map_fixed(_errno, buffer, length, iova) \
194+ ({ \
195+ __u64 __iova = iova; \
196+ EXPECT_ERRNO(_errno, \
197+ _test_ioctl_ioas_map( \
198+ self->fd, self->ioas_id, buffer, length, \
199+ &__iova, \
200+ IOMMU_IOAS_MAP_FIXED_IOVA | \
201+ IOMMU_IOAS_MAP_WRITEABLE | \
202+ IOMMU_IOAS_MAP_READABLE)); \
203+ })
204+
205+static int _test_ioctl_ioas_unmap(int fd, unsigned int ioas_id, uint64_t iova,
206+ size_t length, uint64_t *out_len)
207+{
208+ struct iommu_ioas_unmap cmd = {
209+ .size = sizeof(cmd),
210+ .ioas_id = ioas_id,
211+ .iova = iova,
212+ .length = length,
213+ };
214+ int ret;
215+
216+ ret = ioctl(fd, IOMMU_IOAS_UNMAP, &cmd);
217+ if (out_len)
218+ *out_len = cmd.length;
219+ return ret;
220+}
221+#define test_ioctl_ioas_unmap(iova, length) \
222+ ASSERT_EQ(0, _test_ioctl_ioas_unmap(self->fd, self->ioas_id, iova, \
223+ length, NULL))
224+
225+#define test_ioctl_ioas_unmap_id(ioas_id, iova, length) \
226+ ASSERT_EQ(0, _test_ioctl_ioas_unmap(self->fd, ioas_id, iova, length, \
227+ NULL))
228+
229+#define test_err_ioctl_ioas_unmap(_errno, iova, length) \
230+ EXPECT_ERRNO(_errno, _test_ioctl_ioas_unmap(self->fd, self->ioas_id, \
231+ iova, length, NULL))
232+
233+static int _test_ioctl_set_temp_memory_limit(int fd, unsigned int limit)
234+{
235+ struct iommu_test_cmd memlimit_cmd = {
236+ .size = sizeof(memlimit_cmd),
237+ .op = IOMMU_TEST_OP_SET_TEMP_MEMORY_LIMIT,
238+ .memory_limit = { .limit = limit },
239+ };
240+
241+ return ioctl(fd, _IOMMU_TEST_CMD(IOMMU_TEST_OP_SET_TEMP_MEMORY_LIMIT),
242+ &memlimit_cmd);
243+}
244+
245+#define test_ioctl_set_temp_memory_limit(limit) \
246+ ASSERT_EQ(0, _test_ioctl_set_temp_memory_limit(self->fd, limit))
247+
248+#define test_ioctl_set_default_memory_limit() \
249+ test_ioctl_set_temp_memory_limit(65536)
250+
251+static void teardown_iommufd(int fd, struct __test_metadata *_metadata)
252+{
253+ struct iommu_test_cmd test_cmd = {
254+ .size = sizeof(test_cmd),
255+ .op = IOMMU_TEST_OP_MD_CHECK_REFS,
256+ .check_refs = { .length = BUFFER_SIZE,
257+ .uptr = (uintptr_t)buffer },
258+ };
259+
260+ if (fd == -1)
261+ return;
262+
263+ EXPECT_EQ(0, close(fd));
264+
265+ fd = open("/dev/iommu", O_RDWR);
266+ EXPECT_NE(-1, fd);
267+ EXPECT_EQ(0, ioctl(fd, _IOMMU_TEST_CMD(IOMMU_TEST_OP_MD_CHECK_REFS),
268+ &test_cmd));
269+ EXPECT_EQ(0, close(fd));
270+}
271+
272+#define EXPECT_ERRNO(expected_errno, cmd) \
273+ ({ \
274+ ASSERT_EQ(-1, cmd); \
275+ EXPECT_EQ(expected_errno, errno); \
276+ })
277+
278+#endif