Merge tag 'for-linus-iommufd' of git://git.kernel.org/pub/scm/linux/kernel/git/jgg/iommufd

Pull iommufd implementation from Jason Gunthorpe: "iommufd is the user API to control the IOMMU subsystem as it relates to managing IO page tables that point at user space memory. It takes over from drivers/vfio/vfio_iommu_type1.c (aka the VFIO container) which is the VFIO specific interface for a similar idea. We see a broad need for extended features, some being highly IOMMU device specific: - Binding iommu_domain's to PASID/SSID - Userspace IO page tables, for ARM, x86 and S390 - Kernel bypassed invalidation of user page tables - Re-use of the KVM page table in the IOMMU - Dirty page tracking in the IOMMU - Runtime Increase/Decrease of IOPTE size - PRI support with faults resolved in userspace Many of these HW features exist to support VM use cases - for instance the combination of PASID, PRI and Userspace IO Page Tables allows an implementation of DMA Shared Virtual Addressing (vSVA) within a guest. Dirty tracking enables VM live migration with SRIOV devices and PASID support allow creating "scalable IOV" devices, among other things. As these features are fundamental to a VM platform they need to be uniformly exposed to all the driver families that do DMA into VMs, which is currently VFIO and VDPA" For more background, see the extended explanations in Jason's pull request: https://lore.kernel.org/lkml/Y5dzTU8dlmXTbzoJ@nvidia.com/ * tag 'for-linus-iommufd' of git://git.kernel.org/pub/scm/linux/kernel/git/jgg/iommufd: (62 commits) iommufd: Change the order of MSI setup iommufd: Improve a few unclear bits of code iommufd: Fix comment typos vfio: Move vfio group specific code into group.c vfio: Refactor dma APIs for emulated devices vfio: Wrap vfio group module init/clean code into helpers vfio: Refactor vfio_device open and close vfio: Make vfio_device_open() truly device specific vfio: Swap order of vfio_device_container_register() and open_device() vfio: Set device->group in helper function vfio: Create wrappers for group register/unregister vfio: Move the sanity check of the group to vfio_create_group() vfio: Simplify vfio_create_group() iommufd: Allow iommufd to supply /dev/vfio/vfio vfio: Make vfio_container optionally compiled vfio: Move container related MODULE_ALIAS statements into container.c vfio-iommufd: Support iommufd for emulated VFIO devices vfio-iommufd: Support iommufd for physical VFIO devices vfio-iommufd: Allow iommufd to be used in place of a container fd vfio: Use IOMMU_CAP_ENFORCE_CACHE_COHERENCY for vfio_file_enforced_coherent() ...

Linus Torvalds committed 3 years ago

Commit 08cdc212 parents aa5ad10 d6c55c083 files · +12804 −1466Browse files

Unified · Split

83 files

.clang-format +3 −0
Documentation/userspace-api/index.rst +1 −0
Documentation/userspace-api/ioctl/ioctl-number.rst +1 −0
Documentation/userspace-api/iommufd.rst +223 −0
MAINTAINERS +12 −0
drivers/dma/idxd/cdev.c +1 −2
drivers/dma/idxd/init.c +1 −24
drivers/gpu/drm/i915/gvt/kvmgt.c +17 −4
drivers/iommu/Kconfig +1 −0
drivers/iommu/Makefile +2 −2
drivers/iommu/amd/iommu.c +4 −10
drivers/iommu/arm/arm-smmu-v3/arm-smmu-v3-sva.c +62 −42
drivers/iommu/arm/arm-smmu-v3/arm-smmu-v3.c +18 −14
drivers/iommu/arm/arm-smmu-v3/arm-smmu-v3.h +10 −13
drivers/iommu/arm/arm-smmu/arm-smmu.c +0 −3
drivers/iommu/arm/arm-smmu/qcom_iommu.c +1 −6
drivers/iommu/fsl_pamu.c +1 −1
drivers/iommu/fsl_pamu_domain.c +2 −2
drivers/iommu/intel/dmar.c +7 −0
drivers/iommu/intel/iommu.c +40 −16
drivers/iommu/intel/iommu.h +12 −6
drivers/iommu/intel/pasid.c +4 −2
drivers/iommu/intel/svm.c +57 −88
drivers/iommu/io-pgfault.c +17 −60
drivers/iommu/iommu-sva-lib.c +0 −71
drivers/iommu/iommu-sva.c +240 −0
drivers/iommu/iommu-sva.h +11 −3
drivers/iommu/iommu.c +297 −119
drivers/iommu/iommufd/Kconfig +44 −0
drivers/iommu/iommufd/Makefile +13 −0
drivers/iommu/iommufd/device.c +772 −0
drivers/iommu/iommufd/double_span.h +53 −0
drivers/iommu/iommufd/hw_pagetable.c +57 −0
drivers/iommu/iommufd/io_pagetable.c +1216 −0
drivers/iommu/iommufd/io_pagetable.h +241 −0
drivers/iommu/iommufd/ioas.c +398 −0
drivers/iommu/iommufd/iommufd_private.h +307 −0
drivers/iommu/iommufd/iommufd_test.h +93 −0
drivers/iommu/iommufd/main.c +460 −0
drivers/iommu/iommufd/pages.c +1977 −0
drivers/iommu/iommufd/selftest.c +853 −0
drivers/iommu/iommufd/vfio_compat.c +472 −0
drivers/iommu/ipmmu-vmsa.c +0 −2
drivers/iommu/mtk_iommu.c +2 −2
drivers/iommu/omap-iommu.c +3 −3
drivers/iommu/sprd-iommu.c +1 −3
drivers/iommu/tegra-gart.c +1 −1
drivers/iommu/virtio-iommu.c +3 −4
drivers/misc/uacce/uacce.c +1 −1
drivers/pci/ats.c +3 −0
drivers/s390/cio/vfio_ccw_ops.c +3 −0
drivers/s390/crypto/vfio_ap_ops.c +20 −1
drivers/vfio/Kconfig +25 −11
drivers/vfio/Makefile +4 −2
drivers/vfio/container.c +42 −103
drivers/vfio/fsl-mc/vfio_fsl_mc.c +3 −0
drivers/vfio/group.c +877 −0
drivers/vfio/iommufd.c +158 −0
drivers/vfio/pci/hisilicon/hisi_acc_vfio_pci.c +6 −0
drivers/vfio/pci/mlx5/main.c +3 −0
drivers/vfio/pci/vfio_pci.c +3 −0
drivers/vfio/platform/vfio_amba.c +3 −0
drivers/vfio/platform/vfio_platform.c +3 −0
drivers/vfio/vfio.h +114 −6
drivers/vfio/vfio_main.c +193 −790
include/linux/intel-svm.h +0 −13
include/linux/interval_tree.h +58 −0
include/linux/iommu.h +114 −31
include/linux/iommufd.h +98 −0
include/linux/sched/user.h +1 −1
include/linux/vfio.h +39 −0
include/uapi/linux/iommufd.h +347 −0
kernel/user.c +1 −0
lib/Kconfig +4 −0
lib/interval_tree.c +132 −0
scripts/kernel-doc +8 −4
tools/testing/selftests/Makefile +1 −0
tools/testing/selftests/iommu/.gitignore +3 −0
tools/testing/selftests/iommu/Makefile +12 −0
tools/testing/selftests/iommu/config +2 −0
tools/testing/selftests/iommu/iommufd.c +1654 −0
tools/testing/selftests/iommu/iommufd_fail_nth.c +580 −0
tools/testing/selftests/iommu/iommufd_utils.h +278 −0

.clang-format+3−0

   - 'inet_lhash2_for_each_icsk'
   - 'inet_lhash2_for_each_icsk_continue'
   - 'inet_lhash2_for_each_icsk_rcu'
+  - 'interval_tree_for_each_double_span'
+  - 'interval_tree_for_each_span'
   - 'intlist__for_each_entry'
   - 'intlist__for_each_entry_safe'
+  - 'iopt_for_each_contig_area'
   - 'kcore_copy__for_each_phdr'
   - 'key_for_each'
   - 'key_for_each_safe'

Documentation/userspace-api/index.rst+1−0

@@ -25,6 +25,7 @@ place where this information is gathered. · 24 unmodified lines
25	ebpf/index	25	ebpf/index
26	ioctl/index	26	ioctl/index
27	iommu	27	iommu
		28	iommufd
28	media/index	29	media/index
29	netlink/index	30	netlink/index
30	sysfs-platform_profile	31	sysfs-platform_profile

Documentation/userspace-api/ioctl/ioctl-number.rst+1−0

@@ -105,6 +105,7 @@ Code Seq# Include File Comments · 104 unmodified lines
105	'8' all SNP8023 advanced NIC card	105	'8' all SNP8023 advanced NIC card
106	<mailto:mcr@solidum.com>	106	<mailto:mcr@solidum.com>
107	';' 64-7F linux/vfio.h	107	';' 64-7F linux/vfio.h
		108	';' 80-FF linux/iommufd.h
108	'=' 00-3f uapi/linux/ptp_clock.h <mailto:richardcochran@gmail.com>	109	'=' 00-3f uapi/linux/ptp_clock.h <mailto:richardcochran@gmail.com>
109	'@' 00-0F linux/radeonfb.h conflict!	110	'@' 00-0F linux/radeonfb.h conflict!
110	'@' 00-0F drivers/video/aty/aty128fb.c conflict!	111	'@' 00-0F drivers/video/aty/aty128fb.c conflict!

Documentation/userspace-api/iommufd.rst+223−0

@@ -0,0 +1,223 @@
	1	.. SPDX-License-Identifier: GPL-2.0+
	2
	3	=======
	4	IOMMUFD
	5	=======
	6
	7	:Author: Jason Gunthorpe
	8	:Author: Kevin Tian
	9
	10	Overview
	11	========
	12
	13	IOMMUFD is the user API to control the IOMMU subsystem as it relates to managing
	14	IO page tables from userspace using file descriptors. It intends to be general
	15	and consumable by any driver that wants to expose DMA to userspace. These
	16	drivers are eventually expected to deprecate any internal IOMMU logic
	17	they may already/historically implement (e.g. vfio_iommu_type1.c).
	18
	19	At minimum iommufd provides universal support of managing I/O address spaces and
	20	I/O page tables for all IOMMUs, with room in the design to add non-generic
	21	features to cater to specific hardware functionality.
	22
	23	In this context the capital letter (IOMMUFD) refers to the subsystem while the
	24	small letter (iommufd) refers to the file descriptors created via /dev/iommu for
	25	use by userspace.
	26
	27	Key Concepts
	28	============
	29
	30	User Visible Objects
	31	--------------------
	32
	33	Following IOMMUFD objects are exposed to userspace:
	34
	35	- IOMMUFD_OBJ_IOAS, representing an I/O address space (IOAS), allowing map/unmap
	36	of user space memory into ranges of I/O Virtual Address (IOVA).
	37
	38	The IOAS is a functional replacement for the VFIO container, and like the VFIO
	39	container it copies an IOVA map to a list of iommu_domains held within it.
	40
	41	- IOMMUFD_OBJ_DEVICE, representing a device that is bound to iommufd by an
	42	external driver.
	43
	44	- IOMMUFD_OBJ_HW_PAGETABLE, representing an actual hardware I/O page table
	45	(i.e. a single struct iommu_domain) managed by the iommu driver.
	46
	47	The IOAS has a list of HW_PAGETABLES that share the same IOVA mapping and
	48	it will synchronize its mapping with each member HW_PAGETABLE.
	49
	50	All user-visible objects are destroyed via the IOMMU_DESTROY uAPI.
	51
	52	The diagram below shows relationship between user-visible objects and kernel
	53	datastructures (external to iommufd), with numbers referred to operations
	54	creating the objects and links::
	55
	56	_________________________________________________________
	57	\| iommufd \|
	58	\| [1] \|
	59	\| _________________ \|
	60	\| \| \| \|
	61	\| \| \| \|
	62	\| \| \| \|
	63	\| \| \| \|
	64	\| \| \| \|
	65	\| \| \| \|
	66	\| \| \| [3] [2] \|
	67	\| \| \| ____________ __________ \|
	68	\| \| IOAS \|<--\| \|<------\| \| \|
	69	\| \| \| \|HW_PAGETABLE\| \| DEVICE \| \|
	70	\| \| \| \|____________\| \|__________\| \|
	71	\| \| \| \| \| \|
	72	\| \| \| \| \| \|
	73	\| \| \| \| \| \|
	74	\| \| \| \| \| \|
	75	\| \| \| \| \| \|
	76	\| \|_________________\| \| \| \|
	77	\| \| \| \| \|
	78	\|_________\|___________________\|___________________\|_______\|
	79	\| \| \|
	80	\| _____v______ _______v_____
	81	\| PFN storage \| \| \| \|
	82	\|------------>\|iommu_domain\| \|struct device\|
	83	\|____________\| \|_____________\|
	84
	85	1. IOMMUFD_OBJ_IOAS is created via the IOMMU_IOAS_ALLOC uAPI. An iommufd can
	86	hold multiple IOAS objects. IOAS is the most generic object and does not
	87	expose interfaces that are specific to single IOMMU drivers. All operations
	88	on the IOAS must operate equally on each of the iommu_domains inside of it.
	89
	90	2. IOMMUFD_OBJ_DEVICE is created when an external driver calls the IOMMUFD kAPI
	91	to bind a device to an iommufd. The driver is expected to implement a set of
	92	ioctls to allow userspace to initiate the binding operation. Successful
	93	completion of this operation establishes the desired DMA ownership over the
	94	device. The driver must also set the driver_managed_dma flag and must not
	95	touch the device until this operation succeeds.
	96
	97	3. IOMMUFD_OBJ_HW_PAGETABLE is created when an external driver calls the IOMMUFD
	98	kAPI to attach a bound device to an IOAS. Similarly the external driver uAPI
	99	allows userspace to initiate the attaching operation. If a compatible
	100	pagetable already exists then it is reused for the attachment. Otherwise a
	101	new pagetable object and iommu_domain is created. Successful completion of
	102	this operation sets up the linkages among IOAS, device and iommu_domain. Once
	103	this completes the device could do DMA.
	104
	105	Every iommu_domain inside the IOAS is also represented to userspace as a
	106	HW_PAGETABLE object.
	107
	108	.. note::
	109
	110	Future IOMMUFD updates will provide an API to create and manipulate the
	111	HW_PAGETABLE directly.
	112
	113	A device can only bind to an iommufd due to DMA ownership claim and attach to at
	114	most one IOAS object (no support of PASID yet).
	115
	116	Kernel Datastructure
	117	--------------------
	118
	119	User visible objects are backed by following datastructures:
	120
	121	- iommufd_ioas for IOMMUFD_OBJ_IOAS.
	122	- iommufd_device for IOMMUFD_OBJ_DEVICE.
	123	- iommufd_hw_pagetable for IOMMUFD_OBJ_HW_PAGETABLE.
	124
	125	Several terminologies when looking at these datastructures:
	126
	127	- Automatic domain - refers to an iommu domain created automatically when
	128	attaching a device to an IOAS object. This is compatible to the semantics of
	129	VFIO type1.
	130
	131	- Manual domain - refers to an iommu domain designated by the user as the
	132	target pagetable to be attached to by a device. Though currently there are
	133	no uAPIs to directly create such domain, the datastructure and algorithms
	134	are ready for handling that use case.
	135
	136	- In-kernel user - refers to something like a VFIO mdev that is using the
	137	IOMMUFD access interface to access the IOAS. This starts by creating an
	138	iommufd_access object that is similar to the domain binding a physical device
	139	would do. The access object will then allow converting IOVA ranges into struct
	140	page * lists, or doing direct read/write to an IOVA.
	141
	142	iommufd_ioas serves as the metadata datastructure to manage how IOVA ranges are
	143	mapped to memory pages, composed of:
	144
	145	- struct io_pagetable holding the IOVA map
	146	- struct iopt_area's representing populated portions of IOVA
	147	- struct iopt_pages representing the storage of PFNs
	148	- struct iommu_domain representing the IO page table in the IOMMU
	149	- struct iopt_pages_access representing in-kernel users of PFNs
	150	- struct xarray pinned_pfns holding a list of pages pinned by in-kernel users
	151
	152	Each iopt_pages represents a logical linear array of full PFNs. The PFNs are
	153	ultimately derived from userspace VAs via an mm_struct. Once they have been
	154	pinned the PFNs are stored in IOPTEs of an iommu_domain or inside the pinned_pfns
	155	xarray if they have been pinned through an iommufd_access.
	156
	157	PFN have to be copied between all combinations of storage locations, depending
	158	on what domains are present and what kinds of in-kernel "software access" users
	159	exist. The mechanism ensures that a page is pinned only once.
	160
	161	An io_pagetable is composed of iopt_areas pointing at iopt_pages, along with a
	162	list of iommu_domains that mirror the IOVA to PFN map.
	163
	164	Multiple io_pagetable-s, through their iopt_area-s, can share a single
	165	iopt_pages which avoids multi-pinning and double accounting of page
	166	consumption.
	167
	168	iommufd_ioas is sharable between subsystems, e.g. VFIO and VDPA, as long as
	169	devices managed by different subsystems are bound to a same iommufd.
	170
	171	IOMMUFD User API
	172	================
	173
	174	.. kernel-doc:: include/uapi/linux/iommufd.h
	175
	176	IOMMUFD Kernel API
	177	==================
	178
	179	The IOMMUFD kAPI is device-centric with group-related tricks managed behind the
	180	scene. This allows the external drivers calling such kAPI to implement a simple
	181	device-centric uAPI for connecting its device to an iommufd, instead of
	182	explicitly imposing the group semantics in its uAPI as VFIO does.
	183
	184	.. kernel-doc:: drivers/iommu/iommufd/device.c
	185	:export:
	186
	187	.. kernel-doc:: drivers/iommu/iommufd/main.c
	188	:export:
	189
	190	VFIO and IOMMUFD
	191	----------------
	192
	193	Connecting a VFIO device to iommufd can be done in two ways.
	194
	195	First is a VFIO compatible way by directly implementing the /dev/vfio/vfio
	196	container IOCTLs by mapping them into io_pagetable operations. Doing so allows
	197	the use of iommufd in legacy VFIO applications by symlinking /dev/vfio/vfio to
	198	/dev/iommufd or extending VFIO to SET_CONTAINER using an iommufd instead of a
	199	container fd.
	200
	201	The second approach directly extends VFIO to support a new set of device-centric
	202	user API based on aforementioned IOMMUFD kernel API. It requires userspace
	203	change but better matches the IOMMUFD API semantics and easier to support new
	204	iommufd features when comparing it to the first approach.
	205
	206	Currently both approaches are still work-in-progress.
	207
	208	There are still a few gaps to be resolved to catch up with VFIO type1, as
	209	documented in iommufd_vfio_check_extension().
	210
	211	Future TODOs
	212	============
	213
	214	Currently IOMMUFD supports only kernel-managed I/O page table, similar to VFIO
	215	type1. New features on the radar include:
	216
	217	- Binding iommu_domain's to PASID/SSID
	218	- Userspace page tables, for ARM, x86 and S390
	219	- Kernel bypass'd invalidation of user page tables
	220	- Re-use of the KVM page table in the IOMMU
	221	- Dirty page tracking in the IOMMU
	222	- Runtime Increase/Decrease of IOPTE size
	223	- PRI support with faults resolved in userspace

MAINTAINERS+12−0

@@ -10793,6 +10793,18 @@ F: drivers/iommu/dma-iommu.h · 10792 unmodified lines
10793	F: drivers/iommu/iova.c	10793	F: drivers/iommu/iova.c
10794	F: include/linux/iova.h	10794	F: include/linux/iova.h
10795		10795
		10796	IOMMUFD
		10797	M: Jason Gunthorpe <jgg@nvidia.com>
		10798	M: Kevin Tian <kevin.tian@intel.com>
		10799	L: iommu@lists.linux.dev
		10800	S: Maintained
		10801	T: git git://git.kernel.org/pub/scm/linux/kernel/git/jgg/iommufd.git
		10802	F: Documentation/userspace-api/iommufd.rst
		10803	F: drivers/iommu/iommufd/
		10804	F: include/linux/iommufd.h
		10805	F: include/uapi/linux/iommufd.h
		10806	F: tools/testing/selftests/iommu/
		10807
10796	IOMMU SUBSYSTEM	10808	IOMMU SUBSYSTEM
10797	M: Joerg Roedel <joro@8bytes.org>	10809	M: Joerg Roedel <joro@8bytes.org>
10798	M: Will Deacon <will@kernel.org>	10810	M: Will Deacon <will@kernel.org>

drivers/dma/idxd/cdev.c+1−2

 #include <linux/pci.h>
 #include <linux/device.h>
 #include <linux/sched/task.h>
-#include <linux/intel-svm.h>
 #include <linux/io-64-nonatomic-lo-hi.h>
 #include <linux/cdev.h>
 #include <linux/fs.h>
 	filp->private_data = ctx;
 	if (device_user_pasid_enabled(idxd)) {
-		sva = iommu_sva_bind_device(dev, current->mm, NULL);
+		sva = iommu_sva_bind_device(dev, current->mm);
 		if (IS_ERR(sva)) {
 			rc = PTR_ERR(sva);
 			dev_err(dev, "pasid allocation failed: %d\n", rc);

drivers/dma/idxd/init.c+1−24

@@ -14,7 +14,6 @@ · 13 unmodified lines
14	#include <linux/io-64-nonatomic-lo-hi.h>	14	#include <linux/io-64-nonatomic-lo-hi.h>
15	#include <linux/device.h>	15	#include <linux/device.h>
16	#include <linux/idr.h>	16	#include <linux/idr.h>
17	#include <linux/intel-svm.h>
18	#include <linux/iommu.h>	17	#include <linux/iommu.h>
19	#include <uapi/linux/idxd.h>	18	#include <uapi/linux/idxd.h>
20	#include <linux/dmaengine.h>	19	#include <linux/dmaengine.h>
@@ -502,29 +501,7 @@ static struct idxd_device idxd_alloc(struct pci_dev pdev, struct idxd_driver_d · 481 unmodified lines
502		501
503	static int idxd_enable_system_pasid(struct idxd_device *idxd)	502	static int idxd_enable_system_pasid(struct idxd_device *idxd)
504	{	503	{
505	int flags;	504	return -EOPNOTSUPP;
506	unsigned int pasid;
507	struct iommu_sva *sva;
508
509	flags = SVM_FLAG_SUPERVISOR_MODE;
510
511	sva = iommu_sva_bind_device(&idxd->pdev->dev, NULL, &flags);
512	if (IS_ERR(sva)) {
513	dev_warn(&idxd->pdev->dev,
514	"iommu sva bind failed: %ld\n", PTR_ERR(sva));
515	return PTR_ERR(sva);
516	}
517
518	pasid = iommu_sva_get_pasid(sva);
519	if (pasid == IOMMU_PASID_INVALID) {
520	iommu_sva_unbind_device(sva);
521	return -ENODEV;
522	}
523
524	idxd->sva = sva;
525	idxd->pasid = pasid;
526	dev_dbg(&idxd->pdev->dev, "system pasid: %u\n", pasid);
527	return 0;
528	}	505	}
529		506
530	static void idxd_disable_system_pasid(struct idxd_device *idxd)	507	static void idxd_disable_system_pasid(struct idxd_device *idxd)

drivers/gpu/drm/i915/gvt/kvmgt.c+17−4

 	vgpu->attached = true;
-	kvmgt_protect_table_init(vgpu);
-	gvt_cache_init(vgpu);
 	vgpu->track_node.track_write = kvmgt_page_track_write;
 	vgpu->track_node.track_flush_slot = kvmgt_page_track_flush_slot;
 	kvm_get_kvm(vgpu->vfio_device.kvm);
 	kvmgt_protect_table_destroy(vgpu);
 	gvt_cache_destroy(vgpu);
+	WARN_ON(vgpu->nr_cache_entries);
+	vgpu->gfn_cache = RB_ROOT;
+	vgpu->dma_addr_cache = RB_ROOT;
 	intel_vgpu_release_msi_eventfd_ctx(vgpu);
 	vgpu->attached = false;
 	struct intel_vgpu *vgpu = vfio_dev_to_vgpu(vfio_dev);
 	struct intel_vgpu_type *type =
 		container_of(mdev->type, struct intel_vgpu_type, type);
+	int ret;
 	vgpu->gvt = kdev_to_i915(mdev->type->parent->dev)->gvt;
-	return intel_gvt_create_vgpu(vgpu, type->conf);
+	ret = intel_gvt_create_vgpu(vgpu, type->conf);
+	if (ret)
+		return ret;
+	kvmgt_protect_table_init(vgpu);
+	gvt_cache_init(vgpu);
+	return 0;
 }
 static void intel_vgpu_release_dev(struct vfio_device *vfio_dev)
 	.mmap		= intel_vgpu_mmap,
 	.ioctl		= intel_vgpu_ioctl,
 	.dma_unmap	= intel_vgpu_dma_unmap,
+	.bind_iommufd	= vfio_iommufd_emulated_bind,
+	.unbind_iommufd = vfio_iommufd_emulated_unbind,
+	.attach_ioas	= vfio_iommufd_emulated_attach_ioas,
 };
 static int intel_vgpu_probe(struct mdev_device *mdev)

drivers/iommu/Kconfig+1−0

@@ -188,6 +188,7 @@ config MSM_IOMMU · 187 unmodified lines
188		188
189	source "drivers/iommu/amd/Kconfig"	189	source "drivers/iommu/amd/Kconfig"
190	source "drivers/iommu/intel/Kconfig"	190	source "drivers/iommu/intel/Kconfig"
		191	source "drivers/iommu/iommufd/Kconfig"
191		192
192	config IRQ_REMAP	193	config IRQ_REMAP
193	bool "Support for Interrupt Remapping"	194	bool "Support for Interrupt Remapping"

drivers/iommu/Makefile+2−2

@@ -1,5 +1,5 @@
1	# SPDX-License-Identifier: GPL-2.0	1	# SPDX-License-Identifier: GPL-2.0
2	obj-y += amd/ intel/ arm/	2	obj-y += amd/ intel/ arm/ iommufd/
3	obj-$(CONFIG_IOMMU_API) += iommu.o	3	obj-$(CONFIG_IOMMU_API) += iommu.o
4	obj-$(CONFIG_IOMMU_API) += iommu-traces.o	4	obj-$(CONFIG_IOMMU_API) += iommu-traces.o
5	obj-$(CONFIG_IOMMU_API) += iommu-sysfs.o	5	obj-$(CONFIG_IOMMU_API) += iommu-sysfs.o
@@ -28,6 +28,6 @@ obj-$(CONFIG_FSL_PAMU) += fsl_pamu.o fsl_pamu_domain.o · 22 unmodified lines
28	obj-$(CONFIG_S390_IOMMU) += s390-iommu.o	28	obj-$(CONFIG_S390_IOMMU) += s390-iommu.o
29	obj-$(CONFIG_HYPERV_IOMMU) += hyperv-iommu.o	29	obj-$(CONFIG_HYPERV_IOMMU) += hyperv-iommu.o
30	obj-$(CONFIG_VIRTIO_IOMMU) += virtio-iommu.o	30	obj-$(CONFIG_VIRTIO_IOMMU) += virtio-iommu.o
31	obj-$(CONFIG_IOMMU_SVA) += iommu-sva-lib.o io-pgfault.o	31	obj-$(CONFIG_IOMMU_SVA) += iommu-sva.o io-pgfault.o
32	obj-$(CONFIG_SPRD_IOMMU) += sprd-iommu.o	32	obj-$(CONFIG_SPRD_IOMMU) += sprd-iommu.o
33	obj-$(CONFIG_APPLE_DART) += apple-dart.o	33	obj-$(CONFIG_APPLE_DART) += apple-dart.o

drivers/iommu/amd/iommu.c+4−10

 static int amd_iommu_attach_device(struct iommu_domain *dom,
 				   struct device *dev)
 {
+	struct iommu_dev_data *dev_data = dev_iommu_priv_get(dev);
 	struct protection_domain *domain = to_pdomain(dom);
-	struct iommu_dev_data *dev_data;
+	struct amd_iommu *iommu = rlookup_amd_iommu(dev);
-	struct amd_iommu *iommu;
 	int ret;
-	if (!check_device(dev))
-		return -EINVAL;
-	dev_data = dev_iommu_priv_get(dev);
 	dev_data->defer_attach = false;
-	iommu = rlookup_amd_iommu(dev);
-	if (!iommu)
-		return -EINVAL;
 	if (dev_data->domain)
 		detach_device(dev);
 		return false;
 	case IOMMU_CAP_PRE_BOOT_PROTECTION:
 		return amdr_ivrs_remap_support;
+	case IOMMU_CAP_ENFORCE_CACHE_COHERENCY:
+		return true;
 	default:
 		break;
 	}

drivers/iommu/arm/arm-smmu-v3/arm-smmu-v3-sva.c+62−42

 #include <linux/slab.h>
 #include "arm-smmu-v3.h"
-#include "../../iommu-sva-lib.h"
+#include "../../iommu-sva.h"
 #include "../../io-pgtable-arm.h"
 struct arm_smmu_mmu_notifier {
 	if (!bond)
 		return ERR_PTR(-ENOMEM);
-	/* Allocate a PASID for this mm if necessary */
-	ret = iommu_sva_alloc_pasid(mm, 1, (1U << master->ssid_bits) - 1);
-	if (ret)
-		goto err_free_bond;
 	bond->mm = mm;
 	bond->sva.dev = dev;
 	refcount_set(&bond->refs, 1);
 	return ERR_PTR(ret);
 }
-struct iommu_sva *
-arm_smmu_sva_bind(struct device *dev, struct mm_struct *mm, void *drvdata)
-{
-	struct iommu_sva *handle;
-	struct iommu_domain *domain = iommu_get_domain_for_dev(dev);
-	struct arm_smmu_domain *smmu_domain = to_smmu_domain(domain);
-	if (smmu_domain->stage != ARM_SMMU_DOMAIN_S1)
-		return ERR_PTR(-EINVAL);
-	mutex_lock(&sva_lock);
-	handle = __arm_smmu_sva_bind(dev, mm);
-	mutex_unlock(&sva_lock);
-	return handle;
-}
-void arm_smmu_sva_unbind(struct iommu_sva *handle)
-{
-	struct arm_smmu_bond *bond = sva_to_bond(handle);
-	mutex_lock(&sva_lock);
-	if (refcount_dec_and_test(&bond->refs)) {
-		list_del(&bond->list);
-		arm_smmu_mmu_notifier_put(bond->smmu_mn);
-		kfree(bond);
-	}
-	mutex_unlock(&sva_lock);
-}
-u32 arm_smmu_sva_get_pasid(struct iommu_sva *handle)
-{
-	struct arm_smmu_bond *bond = sva_to_bond(handle);
-	return bond->mm->pasid;
-}
 bool arm_smmu_sva_supported(struct arm_smmu_device *smmu)
 {
 	unsigned long reg, fld;
 	 */
 	mmu_notifier_synchronize();
 }
+void arm_smmu_sva_remove_dev_pasid(struct iommu_domain *domain,
+				   struct device *dev, ioasid_t id)
+{
+	struct mm_struct *mm = domain->mm;
+	struct arm_smmu_bond *bond = NULL, *t;
+	struct arm_smmu_master *master = dev_iommu_priv_get(dev);
+	mutex_lock(&sva_lock);
+	list_for_each_entry(t, &master->bonds, list) {
+		if (t->mm == mm) {
+			bond = t;
+			break;
+		}
+	}
+	if (!WARN_ON(!bond) && refcount_dec_and_test(&bond->refs)) {
+		list_del(&bond->list);
+		arm_smmu_mmu_notifier_put(bond->smmu_mn);
+		kfree(bond);
+	}
+	mutex_unlock(&sva_lock);
+}
+static int arm_smmu_sva_set_dev_pasid(struct iommu_domain *domain,
+				      struct device *dev, ioasid_t id)
+{
+	int ret = 0;
+	struct iommu_sva *handle;
+	struct mm_struct *mm = domain->mm;
+	mutex_lock(&sva_lock);
+	handle = __arm_smmu_sva_bind(dev, mm);
+	if (IS_ERR(handle))
+		ret = PTR_ERR(handle);
+	mutex_unlock(&sva_lock);
+	return ret;
+}
+static void arm_smmu_sva_domain_free(struct iommu_domain *domain)
+{
+	kfree(domain);
+}
+static const struct iommu_domain_ops arm_smmu_sva_domain_ops = {
+	.set_dev_pasid		= arm_smmu_sva_set_dev_pasid,
+	.free			= arm_smmu_sva_domain_free
+};
+struct iommu_domain *arm_smmu_sva_domain_alloc(void)
+{
+	struct iommu_domain *domain;
+	domain = kzalloc(sizeof(*domain), GFP_KERNEL);
+	if (!domain)
+		return NULL;
+	domain->ops = &arm_smmu_sva_domain_ops;
+	return domain;
+}

drivers/iommu/arm/arm-smmu-v3/arm-smmu-v3.c+18−14

 #include "arm-smmu-v3.h"
 #include "../../dma-iommu.h"
-#include "../../iommu-sva-lib.h"
+#include "../../iommu-sva.h"
 static bool disable_bypass = true;
 module_param(disable_bypass, bool, 0444);
 {
 	struct arm_smmu_domain *smmu_domain;
+	if (type == IOMMU_DOMAIN_SVA)
+		return arm_smmu_sva_domain_alloc();
 	if (type != IOMMU_DOMAIN_UNMANAGED &&
 	    type != IOMMU_DOMAIN_DMA &&
 	    type != IOMMU_DOMAIN_DMA_FQ &&
 			goto out_unlock;
 		}
 	} else if (smmu_domain->smmu != smmu) {
-		dev_err(dev,
+		ret = -EINVAL;
-			"cannot attach to SMMU %s (upstream of %s)\n",
-			dev_name(smmu_domain->smmu->dev),
-			dev_name(smmu->dev));
-		ret = -ENXIO;
 		goto out_unlock;
 	} else if (smmu_domain->stage == ARM_SMMU_DOMAIN_S1 &&
 		   master->ssid_bits != smmu_domain->s1_cfg.s1cdmax) {
-		dev_err(dev,
-			"cannot attach to incompatible domain (%u SSID bits != %u)\n",
-			smmu_domain->s1_cfg.s1cdmax, master->ssid_bits);
 		ret = -EINVAL;
 		goto out_unlock;
 	} else if (smmu_domain->stage == ARM_SMMU_DOMAIN_S1 &&
 		   smmu_domain->stall_enabled != master->stall_enabled) {
-		dev_err(dev, "cannot attach to stall-%s domain\n",
-			smmu_domain->stall_enabled ? "enabled" : "disabled");
 		ret = -EINVAL;
 		goto out_unlock;
 	}
 	return 0;
 }
+static void arm_smmu_remove_dev_pasid(struct device *dev, ioasid_t pasid)
+{
+	struct iommu_domain *domain;
+	domain = iommu_get_domain_for_dev_pasid(dev, pasid, IOMMU_DOMAIN_SVA);
+	if (WARN_ON(IS_ERR(domain)) || !domain)
+		return;
+	arm_smmu_sva_remove_dev_pasid(domain, dev, pasid);
+}
 static struct iommu_ops arm_smmu_ops = {
 	.capable		= arm_smmu_capable,
 	.domain_alloc		= arm_smmu_domain_alloc,
 	.device_group		= arm_smmu_device_group,
 	.of_xlate		= arm_smmu_of_xlate,
 	.get_resv_regions	= arm_smmu_get_resv_regions,
+	.remove_dev_pasid	= arm_smmu_remove_dev_pasid,
 	.dev_enable_feat	= arm_smmu_dev_enable_feature,
 	.dev_disable_feat	= arm_smmu_dev_disable_feature,
-	.sva_bind		= arm_smmu_sva_bind,
-	.sva_unbind		= arm_smmu_sva_unbind,
-	.sva_get_pasid		= arm_smmu_sva_get_pasid,
 	.page_response		= arm_smmu_page_response,
 	.def_domain_type	= arm_smmu_def_domain_type,
 	.pgsize_bitmap		= -1UL, /* Restricted during device attach */
 	/* SID/SSID sizes */
 	smmu->ssid_bits = FIELD_GET(IDR1_SSIDSIZE, reg);
 	smmu->sid_bits = FIELD_GET(IDR1_SIDSIZE, reg);
+	smmu->iommu.max_pasids = 1UL << smmu->ssid_bits;
 	/*
 	 * If the SMMU supports fewer bits than would fill a single L2 stream

drivers/iommu/arm/arm-smmu-v3/arm-smmu-v3.h+10−13

@@ -754,11 +754,10 @@ bool arm_smmu_master_sva_enabled(struct arm_smmu_master *master); · 753 unmodified lines
754	int arm_smmu_master_enable_sva(struct arm_smmu_master *master);	754	int arm_smmu_master_enable_sva(struct arm_smmu_master *master);
755	int arm_smmu_master_disable_sva(struct arm_smmu_master *master);	755	int arm_smmu_master_disable_sva(struct arm_smmu_master *master);
756	bool arm_smmu_master_iopf_supported(struct arm_smmu_master *master);	756	bool arm_smmu_master_iopf_supported(struct arm_smmu_master *master);
757	struct iommu_sva arm_smmu_sva_bind(struct device dev, struct mm_struct *mm,
758	void *drvdata);
759	void arm_smmu_sva_unbind(struct iommu_sva *handle);
760	u32 arm_smmu_sva_get_pasid(struct iommu_sva *handle);
761	void arm_smmu_sva_notifier_synchronize(void);	757	void arm_smmu_sva_notifier_synchronize(void);
		758	struct iommu_domain *arm_smmu_sva_domain_alloc(void);
		759	void arm_smmu_sva_remove_dev_pasid(struct iommu_domain *domain,
		760	struct device *dev, ioasid_t id);
762	#else /* CONFIG_ARM_SMMU_V3_SVA */	761	#else /* CONFIG_ARM_SMMU_V3_SVA */
763	static inline bool arm_smmu_sva_supported(struct arm_smmu_device *smmu)	762	static inline bool arm_smmu_sva_supported(struct arm_smmu_device *smmu)
764	{	763	{
@@ -790,19 +789,17 @@ static inline bool arm_smmu_master_iopf_supported(struct arm_smmu_master *master · 25 unmodified lines
790	return false;	789	return false;
791	}	790	}
792		791
793	static inline struct iommu_sva *	792	static inline void arm_smmu_sva_notifier_synchronize(void) {}
794	arm_smmu_sva_bind(struct device dev, struct mm_struct mm, void *drvdata)	793
		794	static inline struct iommu_domain *arm_smmu_sva_domain_alloc(void)
795	{	795	{
796	return ERR_PTR(-ENODEV);	796	return NULL;
797	}	797	}
798		798
799	static inline void arm_smmu_sva_unbind(struct iommu_sva *handle) {}	799	static inline void arm_smmu_sva_remove_dev_pasid(struct iommu_domain *domain,
800		800	struct device *dev,
801	static inline u32 arm_smmu_sva_get_pasid(struct iommu_sva *handle)	801	ioasid_t id)
802	{	802	{
803	return IOMMU_PASID_INVALID;
804	}	803	}
805
806	static inline void arm_smmu_sva_notifier_synchronize(void) {}
807	#endif /* CONFIG_ARM_SMMU_V3_SVA */	804	#endif /* CONFIG_ARM_SMMU_V3_SVA */
808	#endif /* _ARM_SMMU_V3_H */	805	#endif /* _ARM_SMMU_V3_H */

drivers/iommu/arm/arm-smmu/arm-smmu.c+0−3

 	 * different SMMUs.
 	 */
 	if (smmu_domain->smmu != smmu) {
-		dev_err(dev,
-			"cannot attach to SMMU %s whilst already attached to domain on SMMU %s\n",
-			dev_name(smmu_domain->smmu->dev), dev_name(smmu->dev));
 		ret = -EINVAL;
 		goto rpm_put;
 	}

drivers/iommu/arm/arm-smmu/qcom_iommu.c+1−6

 	 * Sanity check the domain. We don't support domains across
 	 * different IOMMUs.
 	 */
-	if (qcom_domain->iommu != qcom_iommu) {
+	if (qcom_domain->iommu != qcom_iommu)
-		dev_err(dev, "cannot attach to IOMMU %s while already "
-			"attached to domain on IOMMU %s\n",
-			dev_name(qcom_domain->iommu->dev),
-			dev_name(qcom_iommu->dev));
 		return -EINVAL;
-	}
 	return 0;
 }

drivers/iommu/fsl_pamu.c+1−1

@@ -211,7 +211,7 @@ int pamu_config_ppaace(int liodn, u32 omi, u32 stashid, int prot) · 210 unmodified lines
211	ppaace->op_encode.index_ot.omi = omi;	211	ppaace->op_encode.index_ot.omi = omi;
212	} else if (~omi != 0) {	212	} else if (~omi != 0) {
213	pr_debug("bad operation mapping index: %d\n", omi);	213	pr_debug("bad operation mapping index: %d\n", omi);
214	return -EINVAL;	214	return -ENODEV;
215	}	215	}
216		216
217	/* configure stash id */	217	/* configure stash id */

drivers/iommu/fsl_pamu_domain.c+2−2

@@ -258,7 +258,7 @@ static int fsl_pamu_attach_device(struct iommu_domain *domain, · 257 unmodified lines
258	liodn = of_get_property(dev->of_node, "fsl,liodn", &len);	258	liodn = of_get_property(dev->of_node, "fsl,liodn", &len);
259	if (!liodn) {	259	if (!liodn) {
260	pr_debug("missing fsl,liodn property at %pOF\n", dev->of_node);	260	pr_debug("missing fsl,liodn property at %pOF\n", dev->of_node);
261	return -EINVAL;	261	return -ENODEV;
262	}	262	}
263		263
264	spin_lock_irqsave(&dma_domain->domain_lock, flags);	264	spin_lock_irqsave(&dma_domain->domain_lock, flags);
@@ -267,7 +267,7 @@ static int fsl_pamu_attach_device(struct iommu_domain *domain, · 2 unmodified lines
267	if (liodn[i] >= PAACE_NUMBER_ENTRIES) {	267	if (liodn[i] >= PAACE_NUMBER_ENTRIES) {
268	pr_debug("Invalid liodn %d, attach device failed for %pOF\n",	268	pr_debug("Invalid liodn %d, attach device failed for %pOF\n",
269	liodn[i], dev->of_node);	269	liodn[i], dev->of_node);
270	ret = -EINVAL;	270	ret = -ENODEV;
271	break;	271	break;
272	}	272	}
273		273

drivers/iommu/intel/dmar.c+7−0

@@ -1105,6 +1105,13 @@ static int alloc_iommu(struct dmar_drhd_unit *drhd) · 1104 unmodified lines
1105		1105
1106	raw_spin_lock_init(&iommu->register_lock);	1106	raw_spin_lock_init(&iommu->register_lock);
1107		1107
		1108	/*
		1109	* A value of N in PSS field of eCap register indicates hardware
		1110	* supports PASID field of N+1 bits.
		1111	*/
		1112	if (pasid_supported(iommu))
		1113	iommu->iommu.max_pasids = 2UL << ecap_pss(iommu->ecap);
		1114
1108	/*	1115	/*
1109	* This is only for hotplug; at boot time intel_iommu_enabled won't	1116	* This is only for hotplug; at boot time intel_iommu_enabled won't
1110	* be set yet. When intel_iommu_init() runs, it registers the units	1117	* be set yet. When intel_iommu_init() runs, it registers the units

drivers/iommu/intel/iommu.c+40−16

@@ -27,7 +27,7 @@ · 26 unmodified lines
27	#include "iommu.h"	27	#include "iommu.h"
28	#include "../dma-iommu.h"	28	#include "../dma-iommu.h"
29	#include "../irq_remapping.h"	29	#include "../irq_remapping.h"
30	#include "../iommu-sva-lib.h"	30	#include "../iommu-sva.h"
31	#include "pasid.h"	31	#include "pasid.h"
32	#include "cap_audit.h"	32	#include "cap_audit.h"
33		33
@@ -4188,6 +4188,8 @@ static struct iommu_domain *intel_iommu_domain_alloc(unsigned type) · 4154 unmodified lines
4188	return domain;	4188	return domain;
4189	case IOMMU_DOMAIN_IDENTITY:	4189	case IOMMU_DOMAIN_IDENTITY:
4190	return &si_domain->domain;	4190	return &si_domain->domain;
		4191	case IOMMU_DOMAIN_SVA:
		4192	return intel_svm_domain_alloc();
4191	default:	4193	default:
4192	return NULL;	4194	return NULL;
4193	}	4195	}
@@ -4213,19 +4215,15 @@ static int prepare_domain_attach_device(struct iommu_domain *domain, · 19 unmodified lines
4213	return -ENODEV;	4215	return -ENODEV;
4214		4216
4215	if (dmar_domain->force_snooping && !ecap_sc_support(iommu->ecap))	4217	if (dmar_domain->force_snooping && !ecap_sc_support(iommu->ecap))
4216	return -EOPNOTSUPP;	4218	return -EINVAL;
4217		4219
4218	/* check if this iommu agaw is sufficient for max mapped address */	4220	/* check if this iommu agaw is sufficient for max mapped address */
4219	addr_width = agaw_to_width(iommu->agaw);	4221	addr_width = agaw_to_width(iommu->agaw);
4220	if (addr_width > cap_mgaw(iommu->cap))	4222	if (addr_width > cap_mgaw(iommu->cap))
4221	addr_width = cap_mgaw(iommu->cap);	4223	addr_width = cap_mgaw(iommu->cap);
4222		4224
4223	if (dmar_domain->max_addr > (1LL << addr_width)) {	4225	if (dmar_domain->max_addr > (1LL << addr_width))
4224	dev_err(dev, "%s: iommu width (%d) is not "	4226	return -EINVAL;
4225	"sufficient for the mapped address (%llx)\n",
4226	__func__, addr_width, dmar_domain->max_addr);
4227	return -EFAULT;
4228	}
4229	dmar_domain->gaw = addr_width;	4227	dmar_domain->gaw = addr_width;
4230		4228
4231	/*	4229	/*
@@ -4471,14 +4469,20 @@ static bool intel_iommu_enforce_cache_coherency(struct iommu_domain *domain) · 239 unmodified lines
4471		4469
4472	static bool intel_iommu_capable(struct device *dev, enum iommu_cap cap)	4470	static bool intel_iommu_capable(struct device *dev, enum iommu_cap cap)
4473	{	4471	{
4474	if (cap == IOMMU_CAP_CACHE_COHERENCY)	4472	struct device_domain_info *info = dev_iommu_priv_get(dev);
		4473
		4474	switch (cap) {
		4475	case IOMMU_CAP_CACHE_COHERENCY:
4475	return true;	4476	return true;
4476	if (cap == IOMMU_CAP_INTR_REMAP)	4477	case IOMMU_CAP_INTR_REMAP:
4477	return irq_remapping_enabled == 1;	4478	return irq_remapping_enabled == 1;
4478	if (cap == IOMMU_CAP_PRE_BOOT_PROTECTION)	4479	case IOMMU_CAP_PRE_BOOT_PROTECTION:
4479	return dmar_platform_optin();	4480	return dmar_platform_optin();
4480		4481	case IOMMU_CAP_ENFORCE_CACHE_COHERENCY:
4481	return false;	4482	return ecap_sc_support(info->iommu->ecap);
		4483	default:
		4484	return false;
		4485	}
4482	}	4486	}
4483		4487
4484	static struct iommu_device intel_iommu_probe_device(struct device dev)	4488	static struct iommu_device intel_iommu_probe_device(struct device dev)
@@ -4732,6 +4736,28 @@ static void intel_iommu_iotlb_sync_map(struct iommu_domain *domain, · 247 unmodified lines
4732	__mapping_notify_one(info->iommu, dmar_domain, pfn, pages);	4736	__mapping_notify_one(info->iommu, dmar_domain, pfn, pages);
4733	}	4737	}
4734		4738
		4739	static void intel_iommu_remove_dev_pasid(struct device *dev, ioasid_t pasid)
		4740	{
		4741	struct intel_iommu *iommu = device_to_iommu(dev, NULL, NULL);
		4742	struct iommu_domain *domain;
		4743
		4744	/* Domain type specific cleanup: */
		4745	domain = iommu_get_domain_for_dev_pasid(dev, pasid, 0);
		4746	if (domain) {
		4747	switch (domain->type) {
		4748	case IOMMU_DOMAIN_SVA:
		4749	intel_svm_remove_dev_pasid(dev, pasid);
		4750	break;
		4751	default:
		4752	/* should never reach here */
		4753	WARN_ON(1);
		4754	break;
		4755	}
		4756	}
		4757
		4758	intel_pasid_tear_down_entry(iommu, dev, pasid, false);
		4759	}
		4760
4735	const struct iommu_ops intel_iommu_ops = {	4761	const struct iommu_ops intel_iommu_ops = {
4736	.capable = intel_iommu_capable,	4762	.capable = intel_iommu_capable,
4737	.domain_alloc = intel_iommu_domain_alloc,	4763	.domain_alloc = intel_iommu_domain_alloc,
@@ -4744,11 +4770,9 @@ const struct iommu_ops intel_iommu_ops = { · 6 unmodified lines
4744	.dev_disable_feat = intel_iommu_dev_disable_feat,	4770	.dev_disable_feat = intel_iommu_dev_disable_feat,
4745	.is_attach_deferred = intel_iommu_is_attach_deferred,	4771	.is_attach_deferred = intel_iommu_is_attach_deferred,
4746	.def_domain_type = device_def_domain_type,	4772	.def_domain_type = device_def_domain_type,
		4773	.remove_dev_pasid = intel_iommu_remove_dev_pasid,
4747	.pgsize_bitmap = SZ_4K,	4774	.pgsize_bitmap = SZ_4K,
4748	#ifdef CONFIG_INTEL_IOMMU_SVM	4775	#ifdef CONFIG_INTEL_IOMMU_SVM
4749	.sva_bind = intel_svm_bind,
4750	.sva_unbind = intel_svm_unbind,
4751	.sva_get_pasid = intel_svm_get_pasid,
4752	.page_response = intel_svm_page_response,	4776	.page_response = intel_svm_page_response,
4753	#endif	4777	#endif
4754	.default_domain_ops = &(const struct iommu_domain_ops) {	4778	.default_domain_ops = &(const struct iommu_domain_ops) {

drivers/iommu/intel/iommu.h+12−6

 #define VTD_FLAG_IRQ_REMAP_PRE_ENABLED	(1 << 1)
 #define VTD_FLAG_SVM_CAPABLE		(1 << 2)
-extern int intel_iommu_sm;
 #define sm_supported(iommu)	(intel_iommu_sm && ecap_smts((iommu)->ecap))
 #define pasid_supported(iommu)	(sm_supported(iommu) &&			\
 				 ecap_pasid((iommu)->ecap))
 extern void intel_svm_check(struct intel_iommu *iommu);
 extern int intel_svm_enable_prq(struct intel_iommu *iommu);
 extern int intel_svm_finish_prq(struct intel_iommu *iommu);
-struct iommu_sva *intel_svm_bind(struct device *dev, struct mm_struct *mm,
-				 void *drvdata);
-void intel_svm_unbind(struct iommu_sva *handle);
-u32 intel_svm_get_pasid(struct iommu_sva *handle);
 int intel_svm_page_response(struct device *dev, struct iommu_fault_event *evt,
 			    struct iommu_page_response *msg);
+struct iommu_domain *intel_svm_domain_alloc(void);
+void intel_svm_remove_dev_pasid(struct device *dev, ioasid_t pasid);
 struct intel_svm_dev {
 	struct list_head list;
 };
 #else
 static inline void intel_svm_check(struct intel_iommu *iommu) {}
+static inline struct iommu_domain *intel_svm_domain_alloc(void)
+{
+	return NULL;
+}
+static inline void intel_svm_remove_dev_pasid(struct device *dev, ioasid_t pasid)
+{
+}
 #endif
 #ifdef CONFIG_INTEL_IOMMU_DEBUGFS
 extern const struct iommu_ops intel_iommu_ops;
 #ifdef CONFIG_INTEL_IOMMU
+extern int intel_iommu_sm;
 extern int iommu_calculate_agaw(struct intel_iommu *iommu);
 extern int iommu_calculate_max_sagaw(struct intel_iommu *iommu);
 extern int dmar_disabled;
 }
 #define dmar_disabled	(1)
 #define intel_iommu_enabled (0)
+#define intel_iommu_sm (0)
 #endif
 static inline const char *decode_prq_descriptor(char *str, size_t size,

drivers/iommu/intel/pasid.c+4−2

 	might_sleep();
 	info = dev_iommu_priv_get(dev);
-	if (WARN_ON(!info || !dev_is_pci(dev) || info->pasid_table))
+	if (WARN_ON(!info || !dev_is_pci(dev)))
-		return -EINVAL;
+		return -ENODEV;
+	if (WARN_ON(info->pasid_table))
+		return -EEXIST;
 	pasid_table = kzalloc(sizeof(*pasid_table), GFP_KERNEL);
 	if (!pasid_table)

drivers/iommu/intel/svm.c+57−88

 #include "iommu.h"
 #include "pasid.h"
 #include "perf.h"
-#include "../iommu-sva-lib.h"
+#include "../iommu-sva.h"
 #include "trace.h"
 static irqreturn_t prq_event_thread(int irq, void *d);
 	return 0;
 }
-static int intel_svm_alloc_pasid(struct device *dev, struct mm_struct *mm,
-				 unsigned int flags)
-{
-	ioasid_t max_pasid = dev_is_pci(dev) ?
-			pci_max_pasids(to_pci_dev(dev)) : intel_pasid_max_id;
-	return iommu_sva_alloc_pasid(mm, PASID_MIN, max_pasid - 1);
-}
 static struct iommu_sva *intel_svm_bind_mm(struct intel_iommu *iommu,
 					   struct device *dev,
-					   struct mm_struct *mm,
+					   struct mm_struct *mm)
-					   unsigned int flags)
 {
 	struct device_domain_info *info = dev_iommu_priv_get(dev);
 	struct intel_svm_dev *sdev;
 		svm->pasid = mm->pasid;
 		svm->mm = mm;
-		svm->flags = flags;
 		INIT_LIST_HEAD_RCU(&svm->devs);
-		if (!(flags & SVM_FLAG_SUPERVISOR_MODE)) {
+		svm->notifier.ops = &intel_mmuops;
-			svm->notifier.ops = &intel_mmuops;
+		ret = mmu_notifier_register(&svm->notifier, mm);
-			ret = mmu_notifier_register(&svm->notifier, mm);
+		if (ret) {
-			if (ret) {
+			kfree(svm);
-				kfree(svm);
+			return ERR_PTR(ret);
-				return ERR_PTR(ret);
-			}
 		}
 		ret = pasid_private_add(svm->pasid, svm);
 		if (ret) {
-			if (svm->notifier.ops)
+			mmu_notifier_unregister(&svm->notifier, mm);
-				mmu_notifier_unregister(&svm->notifier, mm);
 			kfree(svm);
 			return ERR_PTR(ret);
 		}
 	}
 	/* Setup the pasid table: */
-	sflags = (flags & SVM_FLAG_SUPERVISOR_MODE) ?
+	sflags = cpu_feature_enabled(X86_FEATURE_LA57) ? PASID_FLAG_FL5LP : 0;
-			PASID_FLAG_SUPERVISOR_MODE : 0;
-	sflags |= cpu_feature_enabled(X86_FEATURE_LA57) ? PASID_FLAG_FL5LP : 0;
 	ret = intel_pasid_setup_first_level(iommu, dev, mm->pgd, mm->pasid,
 					    FLPT_DEFAULT_DID, sflags);
 	if (ret)
 	kfree(sdev);
 free_svm:
 	if (list_empty(&svm->devs)) {
-		if (svm->notifier.ops)
+		mmu_notifier_unregister(&svm->notifier, mm);
-			mmu_notifier_unregister(&svm->notifier, mm);
 		pasid_private_remove(mm->pasid);
 		kfree(svm);
 	}
 	return IRQ_RETVAL(handled);
 }
-struct iommu_sva *intel_svm_bind(struct device *dev, struct mm_struct *mm, void *drvdata)
-{
-	struct intel_iommu *iommu = device_to_iommu(dev, NULL, NULL);
-	unsigned int flags = 0;
-	struct iommu_sva *sva;
-	int ret;
-	if (drvdata)
-		flags = *(unsigned int *)drvdata;
-	if (flags & SVM_FLAG_SUPERVISOR_MODE) {
-		if (!ecap_srs(iommu->ecap)) {
-			dev_err(dev, "%s: Supervisor PASID not supported\n",
-				iommu->name);
-			return ERR_PTR(-EOPNOTSUPP);
-		}
-		if (mm) {
-			dev_err(dev, "%s: Supervisor PASID with user provided mm\n",
-				iommu->name);
-			return ERR_PTR(-EINVAL);
-		}
-		mm = &init_mm;
-	}
-	mutex_lock(&pasid_mutex);
-	ret = intel_svm_alloc_pasid(dev, mm, flags);
-	if (ret) {
-		mutex_unlock(&pasid_mutex);
-		return ERR_PTR(ret);
-	}
-	sva = intel_svm_bind_mm(iommu, dev, mm, flags);
-	mutex_unlock(&pasid_mutex);
-	return sva;
-}
-void intel_svm_unbind(struct iommu_sva *sva)
-{
-	struct intel_svm_dev *sdev = to_intel_svm_dev(sva);
-	mutex_lock(&pasid_mutex);
-	intel_svm_unbind_mm(sdev->dev, sdev->pasid);
-	mutex_unlock(&pasid_mutex);
-}
-u32 intel_svm_get_pasid(struct iommu_sva *sva)
-{
-	struct intel_svm_dev *sdev;
-	u32 pasid;
-	mutex_lock(&pasid_mutex);
-	sdev = to_intel_svm_dev(sva);
-	pasid = sdev->pasid;
-	mutex_unlock(&pasid_mutex);
-	return pasid;
-}
 int intel_svm_page_response(struct device *dev,
 			    struct iommu_fault_event *evt,
 			    struct iommu_page_response *msg)
 out:
 	return ret;
 }
+void intel_svm_remove_dev_pasid(struct device *dev, ioasid_t pasid)
+{
+	mutex_lock(&pasid_mutex);
+	intel_svm_unbind_mm(dev, pasid);
+	mutex_unlock(&pasid_mutex);
+}
+static int intel_svm_set_dev_pasid(struct iommu_domain *domain,
+				   struct device *dev, ioasid_t pasid)
+{
+	struct device_domain_info *info = dev_iommu_priv_get(dev);
+	struct intel_iommu *iommu = info->iommu;
+	struct mm_struct *mm = domain->mm;
+	struct iommu_sva *sva;
+	int ret = 0;
+	mutex_lock(&pasid_mutex);
+	sva = intel_svm_bind_mm(iommu, dev, mm);
+	if (IS_ERR(sva))
+		ret = PTR_ERR(sva);
+	mutex_unlock(&pasid_mutex);
+	return ret;
+}
+static void intel_svm_domain_free(struct iommu_domain *domain)
+{
+	kfree(to_dmar_domain(domain));
+}
+static const struct iommu_domain_ops intel_svm_domain_ops = {
+	.set_dev_pasid		= intel_svm_set_dev_pasid,
+	.free			= intel_svm_domain_free
+};
+struct iommu_domain *intel_svm_domain_alloc(void)
+{
+	struct dmar_domain *domain;
+	domain = kzalloc(sizeof(*domain), GFP_KERNEL);
+	if (!domain)
+		return NULL;
+	domain->domain.ops = &intel_svm_domain_ops;
+	return &domain->domain;
+}

drivers/iommu/io-pgfault.c+17−60

@@ -11,7 +11,7 @@ · 10 unmodified lines
11	#include <linux/slab.h>	11	#include <linux/slab.h>
12	#include <linux/workqueue.h>	12	#include <linux/workqueue.h>
13		13
14	#include "iommu-sva-lib.h"	14	#include "iommu-sva.h"
15		15
16	/**	16	/**
17	* struct iopf_queue - IO Page Fault queue	17	* struct iopf_queue - IO Page Fault queue
@@ -69,69 +69,18 @@ static int iopf_complete_group(struct device dev, struct iopf_fault iopf, · 51 unmodified lines
69	return iommu_page_response(dev, &resp);	69	return iommu_page_response(dev, &resp);
70	}	70	}
71		71
72	static enum iommu_page_response_code	72	static void iopf_handler(struct work_struct *work)
73	iopf_handle_single(struct iopf_fault *iopf)
74	{
75	vm_fault_t ret;
76	struct mm_struct *mm;
77	struct vm_area_struct *vma;
78	unsigned int access_flags = 0;
79	unsigned int fault_flags = FAULT_FLAG_REMOTE;
80	struct iommu_fault_page_request *prm = &iopf->fault.prm;
81	enum iommu_page_response_code status = IOMMU_PAGE_RESP_INVALID;
82
83	if (!(prm->flags & IOMMU_FAULT_PAGE_REQUEST_PASID_VALID))
84	return status;
85
86	mm = iommu_sva_find(prm->pasid);
87	if (IS_ERR_OR_NULL(mm))
88	return status;
89
90	mmap_read_lock(mm);
91
92	vma = find_extend_vma(mm, prm->addr);
93	if (!vma)
94	/* Unmapped area */
95	goto out_put_mm;
96
97	if (prm->perm & IOMMU_FAULT_PERM_READ)
98	access_flags \|= VM_READ;
99
100	if (prm->perm & IOMMU_FAULT_PERM_WRITE) {
101	access_flags \|= VM_WRITE;
102	fault_flags \|= FAULT_FLAG_WRITE;
103	}
104
105	if (prm->perm & IOMMU_FAULT_PERM_EXEC) {
106	access_flags \|= VM_EXEC;
107	fault_flags \|= FAULT_FLAG_INSTRUCTION;
108	}
109
110	if (!(prm->perm & IOMMU_FAULT_PERM_PRIV))
111	fault_flags \|= FAULT_FLAG_USER;
112
113	if (access_flags & ~vma->vm_flags)
114	/* Access fault */
115	goto out_put_mm;
116
117	ret = handle_mm_fault(vma, prm->addr, fault_flags, NULL);
118	status = ret & VM_FAULT_ERROR ? IOMMU_PAGE_RESP_INVALID :
119	IOMMU_PAGE_RESP_SUCCESS;
120
121	out_put_mm:
122	mmap_read_unlock(mm);
123	mmput(mm);
124
125	return status;
126	}
127
128	static void iopf_handle_group(struct work_struct *work)
129	{	73	{
130	struct iopf_group *group;	74	struct iopf_group *group;
		75	struct iommu_domain *domain;
131	struct iopf_fault iopf, next;	76	struct iopf_fault iopf, next;
132	enum iommu_page_response_code status = IOMMU_PAGE_RESP_SUCCESS;	77	enum iommu_page_response_code status = IOMMU_PAGE_RESP_SUCCESS;
133		78
134	group = container_of(work, struct iopf_group, work);	79	group = container_of(work, struct iopf_group, work);
		80	domain = iommu_get_domain_for_dev_pasid(group->dev,
		81	group->last_fault.fault.prm.pasid, 0);
		82	if (!domain \|\| !domain->iopf_handler)
		83	status = IOMMU_PAGE_RESP_INVALID;
135		84
136	list_for_each_entry_safe(iopf, next, &group->faults, list) {	85	list_for_each_entry_safe(iopf, next, &group->faults, list) {
137	/*	86	/*
@@ -139,7 +88,8 @@ static void iopf_handle_group(struct work_struct *work) · 1 unmodified line
139	* faults in the group if there is an error.	88	* faults in the group if there is an error.
140	*/	89	*/
141	if (status == IOMMU_PAGE_RESP_SUCCESS)	90	if (status == IOMMU_PAGE_RESP_SUCCESS)
142	status = iopf_handle_single(iopf);	91	status = domain->iopf_handler(&iopf->fault,
		92	domain->fault_data);
143		93
144	if (!(iopf->fault.prm.flags &	94	if (!(iopf->fault.prm.flags &
145	IOMMU_FAULT_PAGE_REQUEST_LAST_PAGE))	95	IOMMU_FAULT_PAGE_REQUEST_LAST_PAGE))
@@ -181,6 +131,13 @@ static void iopf_handle_group(struct work_struct *work) · 35 unmodified lines
181	* request completes, outstanding faults will have been dealt with by the time	131	* request completes, outstanding faults will have been dealt with by the time
182	* the PASID is freed.	132	* the PASID is freed.
183	*	133	*
		134	* Any valid page fault will be eventually routed to an iommu domain and the
		135	* page fault handler installed there will get called. The users of this
		136	* handling framework should guarantee that the iommu domain could only be
		137	* freed after the device has stopped generating page faults (or the iommu
		138	* hardware has been set to block the page faults) and the pending page faults
		139	* have been flushed.
		140	*
184	* Return: 0 on success and <0 on error.	141	* Return: 0 on success and <0 on error.
185	*/	142	*/
186	int iommu_queue_iopf(struct iommu_fault fault, void cookie)	143	int iommu_queue_iopf(struct iommu_fault fault, void cookie)
@@ -235,7 +192,7 @@ int iommu_queue_iopf(struct iommu_fault fault, void cookie) · 48 unmodified lines
235	group->last_fault.fault = *fault;	192	group->last_fault.fault = *fault;
236	INIT_LIST_HEAD(&group->faults);	193	INIT_LIST_HEAD(&group->faults);
237	list_add(&group->last_fault.list, &group->faults);	194	list_add(&group->last_fault.list, &group->faults);
238	INIT_WORK(&group->work, iopf_handle_group);	195	INIT_WORK(&group->work, iopf_handler);
239		196
240	/* See if we have partial faults for this group */	197	/* See if we have partial faults for this group */
241	list_for_each_entry_safe(iopf, next, &iopf_param->partial, list) {	198	list_for_each_entry_safe(iopf, next, &iopf_param->partial, list) {

drivers/iommu/iommu-sva-lib.c+0−71

@@ -1,71 +0,0 @@
1	// SPDX-License-Identifier: GPL-2.0
2	/*
3	* Helpers for IOMMU drivers implementing SVA
4	*/
5	#include <linux/mutex.h>
6	#include <linux/sched/mm.h>
7
8	#include "iommu-sva-lib.h"
9
10	static DEFINE_MUTEX(iommu_sva_lock);
11	static DECLARE_IOASID_SET(iommu_sva_pasid);
12
13	/**
14	* iommu_sva_alloc_pasid - Allocate a PASID for the mm
15	* @mm: the mm
16	* @min: minimum PASID value (inclusive)
17	* @max: maximum PASID value (inclusive)
18	*
19	* Try to allocate a PASID for this mm, or take a reference to the existing one
20	* provided it fits within the [@min, @max] range. On success the PASID is
21	* available in mm->pasid and will be available for the lifetime of the mm.
22	*
23	* Returns 0 on success and < 0 on error.
24	*/
25	int iommu_sva_alloc_pasid(struct mm_struct *mm, ioasid_t min, ioasid_t max)
26	{
27	int ret = 0;
28	ioasid_t pasid;
29
30	if (min == INVALID_IOASID \|\| max == INVALID_IOASID \|\|
31	min == 0 \|\| max < min)
32	return -EINVAL;
33
34	mutex_lock(&iommu_sva_lock);
35	/* Is a PASID already associated with this mm? */
36	if (pasid_valid(mm->pasid)) {
37	if (mm->pasid < min \|\| mm->pasid >= max)
38	ret = -EOVERFLOW;
39	goto out;
40	}
41
42	pasid = ioasid_alloc(&iommu_sva_pasid, min, max, mm);
43	if (!pasid_valid(pasid))
44	ret = -ENOMEM;
45	else
46	mm_pasid_set(mm, pasid);
47	out:
48	mutex_unlock(&iommu_sva_lock);
49	return ret;
50	}
51	EXPORT_SYMBOL_GPL(iommu_sva_alloc_pasid);
52
53	/* ioasid_find getter() requires a void * argument */
54	static bool __mmget_not_zero(void *mm)
55	{
56	return mmget_not_zero(mm);
57	}
58
59	/**
60	* iommu_sva_find() - Find mm associated to the given PASID
61	* @pasid: Process Address Space ID assigned to the mm
62	*
63	* On success a reference to the mm is taken, and must be released with mmput().
64	*
65	* Returns the mm corresponding to this PASID, or an error if not found.
66	*/
67	struct mm_struct *iommu_sva_find(ioasid_t pasid)
68	{
69	return ioasid_find(&iommu_sva_pasid, pasid, __mmget_not_zero);
70	}
71	EXPORT_SYMBOL_GPL(iommu_sva_find);

drivers/iommu/iommu-sva.c+240−0

@@ -0,0 +1,240 @@
	1	// SPDX-License-Identifier: GPL-2.0
	2	/*
	3	* Helpers for IOMMU drivers implementing SVA
	4	*/
	5	#include <linux/mutex.h>
	6	#include <linux/sched/mm.h>
	7	#include <linux/iommu.h>
	8
	9	#include "iommu-sva.h"
	10
	11	static DEFINE_MUTEX(iommu_sva_lock);
	12	static DECLARE_IOASID_SET(iommu_sva_pasid);
	13
	14	/**
	15	* iommu_sva_alloc_pasid - Allocate a PASID for the mm
	16	* @mm: the mm
	17	* @min: minimum PASID value (inclusive)
	18	* @max: maximum PASID value (inclusive)
	19	*
	20	* Try to allocate a PASID for this mm, or take a reference to the existing one
	21	* provided it fits within the [@min, @max] range. On success the PASID is
	22	* available in mm->pasid and will be available for the lifetime of the mm.
	23	*
	24	* Returns 0 on success and < 0 on error.
	25	*/
	26	int iommu_sva_alloc_pasid(struct mm_struct *mm, ioasid_t min, ioasid_t max)
	27	{
	28	int ret = 0;
	29	ioasid_t pasid;
	30
	31	if (min == INVALID_IOASID \|\| max == INVALID_IOASID \|\|
	32	min == 0 \|\| max < min)
	33	return -EINVAL;
	34
	35	mutex_lock(&iommu_sva_lock);
	36	/* Is a PASID already associated with this mm? */
	37	if (pasid_valid(mm->pasid)) {
	38	if (mm->pasid < min \|\| mm->pasid >= max)
	39	ret = -EOVERFLOW;
	40	goto out;
	41	}
	42
	43	pasid = ioasid_alloc(&iommu_sva_pasid, min, max, mm);
	44	if (!pasid_valid(pasid))
	45	ret = -ENOMEM;
	46	else
	47	mm_pasid_set(mm, pasid);
	48	out:
	49	mutex_unlock(&iommu_sva_lock);
	50	return ret;
	51	}
	52	EXPORT_SYMBOL_GPL(iommu_sva_alloc_pasid);
	53
	54	/* ioasid_find getter() requires a void * argument */
	55	static bool __mmget_not_zero(void *mm)
	56	{
	57	return mmget_not_zero(mm);
	58	}
	59
	60	/**
	61	* iommu_sva_find() - Find mm associated to the given PASID
	62	* @pasid: Process Address Space ID assigned to the mm
	63	*
	64	* On success a reference to the mm is taken, and must be released with mmput().
	65	*
	66	* Returns the mm corresponding to this PASID, or an error if not found.
	67	*/
	68	struct mm_struct *iommu_sva_find(ioasid_t pasid)
	69	{
	70	return ioasid_find(&iommu_sva_pasid, pasid, __mmget_not_zero);
	71	}
	72	EXPORT_SYMBOL_GPL(iommu_sva_find);
	73
	74	/**
	75	* iommu_sva_bind_device() - Bind a process address space to a device
	76	* @dev: the device
	77	* @mm: the mm to bind, caller must hold a reference to mm_users
	78	*
	79	* Create a bond between device and address space, allowing the device to
	80	* access the mm using the PASID returned by iommu_sva_get_pasid(). If a
	81	* bond already exists between @device and @mm, an additional internal
	82	* reference is taken. Caller must call iommu_sva_unbind_device()
	83	* to release each reference.
	84	*
	85	* iommu_dev_enable_feature(dev, IOMMU_DEV_FEAT_SVA) must be called first, to
	86	* initialize the required SVA features.
	87	*
	88	* On error, returns an ERR_PTR value.
	89	*/
	90	struct iommu_sva iommu_sva_bind_device(struct device dev, struct mm_struct *mm)
	91	{
	92	struct iommu_domain *domain;
	93	struct iommu_sva *handle;
	94	ioasid_t max_pasids;
	95	int ret;
	96
	97	max_pasids = dev->iommu->max_pasids;
	98	if (!max_pasids)
	99	return ERR_PTR(-EOPNOTSUPP);
	100
	101	/* Allocate mm->pasid if necessary. */
	102	ret = iommu_sva_alloc_pasid(mm, 1, max_pasids - 1);
	103	if (ret)
	104	return ERR_PTR(ret);
	105
	106	handle = kzalloc(sizeof(*handle), GFP_KERNEL);
	107	if (!handle)
	108	return ERR_PTR(-ENOMEM);
	109
	110	mutex_lock(&iommu_sva_lock);
	111	/* Search for an existing domain. */
	112	domain = iommu_get_domain_for_dev_pasid(dev, mm->pasid,
	113	IOMMU_DOMAIN_SVA);
	114	if (IS_ERR(domain)) {
	115	ret = PTR_ERR(domain);
	116	goto out_unlock;
	117	}
	118
	119	if (domain) {
	120	domain->users++;
	121	goto out;
	122	}
	123
	124	/* Allocate a new domain and set it on device pasid. */
	125	domain = iommu_sva_domain_alloc(dev, mm);
	126	if (!domain) {
	127	ret = -ENOMEM;
	128	goto out_unlock;
	129	}
	130
	131	ret = iommu_attach_device_pasid(domain, dev, mm->pasid);
	132	if (ret)
	133	goto out_free_domain;
	134	domain->users = 1;
	135	out:
	136	mutex_unlock(&iommu_sva_lock);
	137	handle->dev = dev;
	138	handle->domain = domain;
	139
	140	return handle;
	141
	142	out_free_domain:
	143	iommu_domain_free(domain);
	144	out_unlock:
	145	mutex_unlock(&iommu_sva_lock);
	146	kfree(handle);
	147
	148	return ERR_PTR(ret);
	149	}
	150	EXPORT_SYMBOL_GPL(iommu_sva_bind_device);
	151
	152	/**
	153	* iommu_sva_unbind_device() - Remove a bond created with iommu_sva_bind_device
	154	* @handle: the handle returned by iommu_sva_bind_device()
	155	*
	156	* Put reference to a bond between device and address space. The device should
	157	* not be issuing any more transaction for this PASID. All outstanding page
	158	* requests for this PASID must have been flushed to the IOMMU.
	159	*/
	160	void iommu_sva_unbind_device(struct iommu_sva *handle)
	161	{
	162	struct iommu_domain *domain = handle->domain;
	163	ioasid_t pasid = domain->mm->pasid;
	164	struct device *dev = handle->dev;
	165
	166	mutex_lock(&iommu_sva_lock);
	167	if (--domain->users == 0) {
	168	iommu_detach_device_pasid(domain, dev, pasid);
	169	iommu_domain_free(domain);
	170	}
	171	mutex_unlock(&iommu_sva_lock);
	172	kfree(handle);
	173	}
	174	EXPORT_SYMBOL_GPL(iommu_sva_unbind_device);
	175
	176	u32 iommu_sva_get_pasid(struct iommu_sva *handle)
	177	{
	178	struct iommu_domain *domain = handle->domain;
	179
	180	return domain->mm->pasid;
	181	}
	182	EXPORT_SYMBOL_GPL(iommu_sva_get_pasid);
	183
	184	/*
	185	* I/O page fault handler for SVA
	186	*/
	187	enum iommu_page_response_code
	188	iommu_sva_handle_iopf(struct iommu_fault fault, void data)
	189	{
	190	vm_fault_t ret;
	191	struct vm_area_struct *vma;
	192	struct mm_struct *mm = data;
	193	unsigned int access_flags = 0;
	194	unsigned int fault_flags = FAULT_FLAG_REMOTE;
	195	struct iommu_fault_page_request *prm = &fault->prm;
	196	enum iommu_page_response_code status = IOMMU_PAGE_RESP_INVALID;
	197
	198	if (!(prm->flags & IOMMU_FAULT_PAGE_REQUEST_PASID_VALID))
	199	return status;
	200
	201	if (!mmget_not_zero(mm))
	202	return status;
	203
	204	mmap_read_lock(mm);
	205
	206	vma = find_extend_vma(mm, prm->addr);
	207	if (!vma)
	208	/* Unmapped area */
	209	goto out_put_mm;
	210
	211	if (prm->perm & IOMMU_FAULT_PERM_READ)
	212	access_flags \|= VM_READ;
	213
	214	if (prm->perm & IOMMU_FAULT_PERM_WRITE) {
	215	access_flags \|= VM_WRITE;
	216	fault_flags \|= FAULT_FLAG_WRITE;
	217	}
	218
	219	if (prm->perm & IOMMU_FAULT_PERM_EXEC) {
	220	access_flags \|= VM_EXEC;
	221	fault_flags \|= FAULT_FLAG_INSTRUCTION;
	222	}
	223
	224	if (!(prm->perm & IOMMU_FAULT_PERM_PRIV))
	225	fault_flags \|= FAULT_FLAG_USER;
	226
	227	if (access_flags & ~vma->vm_flags)
	228	/* Access fault */
	229	goto out_put_mm;
	230
	231	ret = handle_mm_fault(vma, prm->addr, fault_flags, NULL);
	232	status = ret & VM_FAULT_ERROR ? IOMMU_PAGE_RESP_INVALID :
	233	IOMMU_PAGE_RESP_SUCCESS;
	234
	235	out_put_mm:
	236	mmap_read_unlock(mm);
	237	mmput(mm);
	238
	239	return status;
	240	}

drivers/iommu/iommu-sva.hrenamed from drivers/iommu/iommu-sva-lib.h+11−3

 /*
  * SVA library for IOMMU drivers
  */
-#ifndef _IOMMU_SVA_LIB_H
+#ifndef _IOMMU_SVA_H
-#define _IOMMU_SVA_LIB_H
+#define _IOMMU_SVA_H
 #include <linux/ioasid.h>
 #include <linux/mm_types.h>
 struct iopf_queue *iopf_queue_alloc(const char *name);
 void iopf_queue_free(struct iopf_queue *queue);
 int iopf_queue_discard_partial(struct iopf_queue *queue);
+enum iommu_page_response_code
+iommu_sva_handle_iopf(struct iommu_fault *fault, void *data);
 #else /* CONFIG_IOMMU_SVA */
 static inline int iommu_queue_iopf(struct iommu_fault *fault, void *cookie)
 {
 	return -ENODEV;
 }
+static inline enum iommu_page_response_code
+iommu_sva_handle_iopf(struct iommu_fault *fault, void *data)
+{
+	return IOMMU_PAGE_RESP_INVALID;
+}
 #endif /* CONFIG_IOMMU_SVA */
-#endif /* _IOMMU_SVA_LIB_H */
+#endif /* _IOMMU_SVA_H */

drivers/iommu/iommu.c+297−119

@@ -21,6 +21,7 @@ · 20 unmodified lines
21	#include <linux/idr.h>	21	#include <linux/idr.h>
22	#include <linux/err.h>	22	#include <linux/err.h>
23	#include <linux/pci.h>	23	#include <linux/pci.h>
		24	#include <linux/pci-ats.h>
24	#include <linux/bitops.h>	25	#include <linux/bitops.h>
25	#include <linux/platform_device.h>	26	#include <linux/platform_device.h>
26	#include <linux/property.h>	27	#include <linux/property.h>
@@ -28,9 +29,12 @@ · 1 unmodified line
28	#include <linux/module.h>	29	#include <linux/module.h>
29	#include <linux/cc_platform.h>	30	#include <linux/cc_platform.h>
30	#include <trace/events/iommu.h>	31	#include <trace/events/iommu.h>
		32	#include <linux/sched/mm.h>
31		33
32	#include "dma-iommu.h"	34	#include "dma-iommu.h"
33		35
		36	#include "iommu-sva.h"
		37
34	static struct kset *iommu_group_kset;	38	static struct kset *iommu_group_kset;
35	static DEFINE_IDA(iommu_group_ida);	39	static DEFINE_IDA(iommu_group_ida);
36		40
@@ -42,6 +46,7 @@ struct iommu_group { · 5 unmodified lines
42	struct kobject kobj;	46	struct kobject kobj;
43	struct kobject *devices_kobj;	47	struct kobject *devices_kobj;
44	struct list_head devices;	48	struct list_head devices;
		49	struct xarray pasid_array;
45	struct mutex mutex;	50	struct mutex mutex;
46	void *iommu_data;	51	void *iommu_data;
47	void (iommu_data_release)(void iommu_data);	52	void (iommu_data_release)(void iommu_data);
@@ -278,6 +283,24 @@ static void dev_iommu_free(struct device *dev) · 230 unmodified lines
278	kfree(param);	283	kfree(param);
279	}	284	}
280		285
		286	static u32 dev_iommu_get_max_pasids(struct device *dev)
		287	{
		288	u32 max_pasids = 0, bits = 0;
		289	int ret;
		290
		291	if (dev_is_pci(dev)) {
		292	ret = pci_max_pasids(to_pci_dev(dev));
		293	if (ret > 0)
		294	max_pasids = ret;
		295	} else {
		296	ret = device_property_read_u32(dev, "pasid-num-bits", &bits);
		297	if (!ret)
		298	max_pasids = 1UL << bits;
		299	}
		300
		301	return min_t(u32, max_pasids, dev->iommu->iommu_dev->max_pasids);
		302	}
		303
281	static int __iommu_probe_device(struct device dev, struct list_head group_list)	304	static int __iommu_probe_device(struct device dev, struct list_head group_list)
282	{	305	{
283	const struct iommu_ops *ops = dev->bus->iommu_ops;	306	const struct iommu_ops *ops = dev->bus->iommu_ops;
@@ -303,6 +326,7 @@ static int __iommu_probe_device(struct device dev, struct list_head group_list · 19 unmodified lines
303	}	326	}
304		327
305	dev->iommu->iommu_dev = iommu_dev;	328	dev->iommu->iommu_dev = iommu_dev;
		329	dev->iommu->max_pasids = dev_iommu_get_max_pasids(dev);
306		330
307	group = iommu_group_get_for_dev(dev);	331	group = iommu_group_get_for_dev(dev);
308	if (IS_ERR(group)) {	332	if (IS_ERR(group)) {
@@ -703,6 +727,7 @@ struct iommu_group *iommu_group_alloc(void) · 394 unmodified lines
703	mutex_init(&group->mutex);	727	mutex_init(&group->mutex);
704	INIT_LIST_HEAD(&group->devices);	728	INIT_LIST_HEAD(&group->devices);
705	INIT_LIST_HEAD(&group->entry);	729	INIT_LIST_HEAD(&group->entry);
		730	xa_init(&group->pasid_array);
706		731
707	ret = ida_alloc(&iommu_group_ida, GFP_KERNEL);	732	ret = ida_alloc(&iommu_group_ida, GFP_KERNEL);
708	if (ret < 0) {	733	if (ret < 0) {
@@ -1912,6 +1937,8 @@ EXPORT_SYMBOL_GPL(iommu_domain_alloc); · 1203 unmodified lines
1912		1937
1913	void iommu_domain_free(struct iommu_domain *domain)	1938	void iommu_domain_free(struct iommu_domain *domain)
1914	{	1939	{
		1940	if (domain->type == IOMMU_DOMAIN_SVA)
		1941	mmdrop(domain->mm);
1915	iommu_put_dma_cookie(domain);	1942	iommu_put_dma_cookie(domain);
1916	domain->ops->free(domain);	1943	domain->ops->free(domain);
1917	}	1944	}
@@ -1949,6 +1976,18 @@ static int __iommu_attach_device(struct iommu_domain *domain, · 31 unmodified lines
1949	return ret;	1976	return ret;
1950	}	1977	}
1951		1978
		1979	/**
		1980	* iommu_attach_device - Attach an IOMMU domain to a device
		1981	* @domain: IOMMU domain to attach
		1982	* @dev: Device that will be attached
		1983	*
		1984	* Returns 0 on success and error code on failure
		1985	*
		1986	* Note that EINVAL can be treated as a soft failure, indicating
		1987	* that certain configuration of the domain is incompatible with
		1988	* the device. In this case attaching a different domain to the
		1989	* device may succeed.
		1990	*/
1952	int iommu_attach_device(struct iommu_domain domain, struct device dev)	1991	int iommu_attach_device(struct iommu_domain domain, struct device dev)
1953	{	1992	{
1954	struct iommu_group *group;	1993	struct iommu_group *group;
@@ -2075,6 +2114,18 @@ static int __iommu_attach_group(struct iommu_domain *domain, · 120 unmodified lines
2075	return ret;	2114	return ret;
2076	}	2115	}
2077		2116
		2117	/**
		2118	* iommu_attach_group - Attach an IOMMU domain to an IOMMU group
		2119	* @domain: IOMMU domain to attach
		2120	* @group: IOMMU group that will be attached
		2121	*
		2122	* Returns 0 on success and error code on failure
		2123	*
		2124	* Note that EINVAL can be treated as a soft failure, indicating
		2125	* that certain configuration of the domain is incompatible with
		2126	* the group. In this case attaching a different domain to the
		2127	* group may succeed.
		2128	*/
2078	int iommu_attach_group(struct iommu_domain domain, struct iommu_group group)	2129	int iommu_attach_group(struct iommu_domain domain, struct iommu_group group)
2079	{	2130	{
2080	int ret;	2131	int ret;
@@ -2726,98 +2777,6 @@ int iommu_dev_disable_feature(struct device *dev, enum iommu_dev_features feat) · 645 unmodified lines
2726	}	2777	}
2727	EXPORT_SYMBOL_GPL(iommu_dev_disable_feature);	2778	EXPORT_SYMBOL_GPL(iommu_dev_disable_feature);
2728		2779
2729	/**
2730	* iommu_sva_bind_device() - Bind a process address space to a device
2731	* @dev: the device
2732	* @mm: the mm to bind, caller must hold a reference to it
2733	* @drvdata: opaque data pointer to pass to bind callback
2734	*
2735	* Create a bond between device and address space, allowing the device to access
2736	* the mm using the returned PASID. If a bond already exists between @device and
2737	* @mm, it is returned and an additional reference is taken. Caller must call
2738	* iommu_sva_unbind_device() to release each reference.
2739	*
2740	* iommu_dev_enable_feature(dev, IOMMU_DEV_FEAT_SVA) must be called first, to
2741	* initialize the required SVA features.
2742	*
2743	* On error, returns an ERR_PTR value.
2744	*/
2745	struct iommu_sva *
2746	iommu_sva_bind_device(struct device dev, struct mm_struct mm, void *drvdata)
2747	{
2748	struct iommu_group *group;
2749	struct iommu_sva *handle = ERR_PTR(-EINVAL);
2750	const struct iommu_ops *ops = dev_iommu_ops(dev);
2751
2752	if (!ops->sva_bind)
2753	return ERR_PTR(-ENODEV);
2754
2755	group = iommu_group_get(dev);
2756	if (!group)
2757	return ERR_PTR(-ENODEV);
2758
2759	/* Ensure device count and domain don't change while we're binding */
2760	mutex_lock(&group->mutex);
2761
2762	/*
2763	* To keep things simple, SVA currently doesn't support IOMMU groups
2764	* with more than one device. Existing SVA-capable systems are not
2765	* affected by the problems that required IOMMU groups (lack of ACS
2766	* isolation, device ID aliasing and other hardware issues).
2767	*/
2768	if (iommu_group_device_count(group) != 1)
2769	goto out_unlock;
2770
2771	handle = ops->sva_bind(dev, mm, drvdata);
2772
2773	out_unlock:
2774	mutex_unlock(&group->mutex);
2775	iommu_group_put(group);
2776
2777	return handle;
2778	}
2779	EXPORT_SYMBOL_GPL(iommu_sva_bind_device);
2780
2781	/**
2782	* iommu_sva_unbind_device() - Remove a bond created with iommu_sva_bind_device
2783	* @handle: the handle returned by iommu_sva_bind_device()
2784	*
2785	* Put reference to a bond between device and address space. The device should
2786	* not be issuing any more transaction for this PASID. All outstanding page
2787	* requests for this PASID must have been flushed to the IOMMU.
2788	*/
2789	void iommu_sva_unbind_device(struct iommu_sva *handle)
2790	{
2791	struct iommu_group *group;
2792	struct device *dev = handle->dev;
2793	const struct iommu_ops *ops = dev_iommu_ops(dev);
2794
2795	if (!ops->sva_unbind)
2796	return;
2797
2798	group = iommu_group_get(dev);
2799	if (!group)
2800	return;
2801
2802	mutex_lock(&group->mutex);
2803	ops->sva_unbind(handle);
2804	mutex_unlock(&group->mutex);
2805
2806	iommu_group_put(group);
2807	}
2808	EXPORT_SYMBOL_GPL(iommu_sva_unbind_device);
2809
2810	u32 iommu_sva_get_pasid(struct iommu_sva *handle)
2811	{
2812	const struct iommu_ops *ops = dev_iommu_ops(handle->dev);
2813
2814	if (!ops->sva_get_pasid)
2815	return IOMMU_PASID_INVALID;
2816
2817	return ops->sva_get_pasid(handle);
2818	}
2819	EXPORT_SYMBOL_GPL(iommu_sva_get_pasid);
2820
2821	/*	2780	/*
2822	* Changes the default domain of an iommu group that has only one device	2781	* Changes the default domain of an iommu group that has only one device
2823	*	2782	*
@@ -3087,7 +3046,8 @@ int iommu_device_use_default_domain(struct device *dev) · 263 unmodified lines
3087		3046
3088	mutex_lock(&group->mutex);	3047	mutex_lock(&group->mutex);
3089	if (group->owner_cnt) {	3048	if (group->owner_cnt) {
3090	if (group->owner \|\| !iommu_is_default_domain(group)) {	3049	if (group->owner \|\| !iommu_is_default_domain(group) \|\|
		3050	!xa_empty(&group->pasid_array)) {
3091	ret = -EBUSY;	3051	ret = -EBUSY;
3092	goto unlock_out;	3052	goto unlock_out;
3093	}	3053	}
@@ -3118,7 +3078,7 @@ void iommu_device_unuse_default_domain(struct device *dev) · 24 unmodified lines
3118	return;	3078	return;
3119		3079
3120	mutex_lock(&group->mutex);	3080	mutex_lock(&group->mutex);
3121	if (!WARN_ON(!group->owner_cnt))	3081	if (!WARN_ON(!group->owner_cnt \|\| !xa_empty(&group->pasid_array)))
3122	group->owner_cnt--;	3082	group->owner_cnt--;
3123		3083
3124	mutex_unlock(&group->mutex);	3084	mutex_unlock(&group->mutex);
@@ -3148,40 +3108,49 @@ static int __iommu_group_alloc_blocking_domain(struct iommu_group *group) · 23 unmodified lines
3148	return 0;	3108	return 0;
3149	}	3109	}
3150		3110
		3111	static int __iommu_take_dma_ownership(struct iommu_group group, void owner)
		3112	{
		3113	int ret;
		3114
		3115	if ((group->domain && group->domain != group->default_domain) \|\|
		3116	!xa_empty(&group->pasid_array))
		3117	return -EBUSY;
		3118
		3119	ret = __iommu_group_alloc_blocking_domain(group);
		3120	if (ret)
		3121	return ret;
		3122	ret = __iommu_group_set_domain(group, group->blocking_domain);
		3123	if (ret)
		3124	return ret;
		3125
		3126	group->owner = owner;
		3127	group->owner_cnt++;
		3128	return 0;
		3129	}
		3130
3151	/**	3131	/**
3152	* iommu_group_claim_dma_owner() - Set DMA ownership of a group	3132	* iommu_group_claim_dma_owner() - Set DMA ownership of a group
3153	* @group: The group.	3133	* @group: The group.
3154	* @owner: Caller specified pointer. Used for exclusive ownership.	3134	* @owner: Caller specified pointer. Used for exclusive ownership.
3155	*	3135	*
3156	* This is to support backward compatibility for vfio which manages	3136	* This is to support backward compatibility for vfio which manages the dma
3157	* the dma ownership in iommu_group level. New invocations on this	3137	* ownership in iommu_group level. New invocations on this interface should be
3158	* interface should be prohibited.	3138	* prohibited. Only a single owner may exist for a group.
3159	*/	3139	*/
3160	int iommu_group_claim_dma_owner(struct iommu_group group, void owner)	3140	int iommu_group_claim_dma_owner(struct iommu_group group, void owner)
3161	{	3141	{
3162	int ret = 0;	3142	int ret = 0;
3163		3143
		3144	if (WARN_ON(!owner))
		3145	return -EINVAL;
		3146
3164	mutex_lock(&group->mutex);	3147	mutex_lock(&group->mutex);
3165	if (group->owner_cnt) {	3148	if (group->owner_cnt) {
3166	ret = -EPERM;	3149	ret = -EPERM;
3167	goto unlock_out;	3150	goto unlock_out;
3168	} else {
3169	if (group->domain && group->domain != group->default_domain) {
3170	ret = -EBUSY;
3171	goto unlock_out;
3172	}
3173
3174	ret = __iommu_group_alloc_blocking_domain(group);
3175	if (ret)
3176	goto unlock_out;
3177
3178	ret = __iommu_group_set_domain(group, group->blocking_domain);
3179	if (ret)
3180	goto unlock_out;
3181	group->owner = owner;
3182	}	3151	}
3183		3152
3184	group->owner_cnt++;	3153	ret = __iommu_take_dma_ownership(group, owner);
3185	unlock_out:	3154	unlock_out:
3186	mutex_unlock(&group->mutex);	3155	mutex_unlock(&group->mutex);
3187		3156
@@ -3190,29 +3159,91 @@ unlock_out: · 2 unmodified lines
3190	EXPORT_SYMBOL_GPL(iommu_group_claim_dma_owner);	3159	EXPORT_SYMBOL_GPL(iommu_group_claim_dma_owner);
3191		3160
3192	/**	3161	/**
3193	* iommu_group_release_dma_owner() - Release DMA ownership of a group	3162	* iommu_device_claim_dma_owner() - Set DMA ownership of a device
3194	* @group: The group.	3163	* @dev: The device.
		3164	* @owner: Caller specified pointer. Used for exclusive ownership.
3195	*	3165	*
3196	* Release the DMA ownership claimed by iommu_group_claim_dma_owner().	3166	* Claim the DMA ownership of a device. Multiple devices in the same group may
		3167	* concurrently claim ownership if they present the same owner value. Returns 0
		3168	* on success and error code on failure
3197	*/	3169	*/
3198	void iommu_group_release_dma_owner(struct iommu_group *group)	3170	int iommu_device_claim_dma_owner(struct device dev, void owner)
3199	{	3171	{
3200	int ret;	3172	struct iommu_group *group = iommu_group_get(dev);
		3173	int ret = 0;
		3174
		3175	if (!group)
		3176	return -ENODEV;
		3177	if (WARN_ON(!owner))
		3178	return -EINVAL;
3201		3179
3202	mutex_lock(&group->mutex);	3180	mutex_lock(&group->mutex);
3203	if (WARN_ON(!group->owner_cnt \|\| !group->owner))	3181	if (group->owner_cnt) {
		3182	if (group->owner != owner) {
		3183	ret = -EPERM;
		3184	goto unlock_out;
		3185	}
		3186	group->owner_cnt++;
3204	goto unlock_out;	3187	goto unlock_out;
		3188	}
		3189
		3190	ret = __iommu_take_dma_ownership(group, owner);
		3191	unlock_out:
		3192	mutex_unlock(&group->mutex);
		3193	iommu_group_put(group);
		3194
		3195	return ret;
		3196	}
		3197	EXPORT_SYMBOL_GPL(iommu_device_claim_dma_owner);
		3198
		3199	static void __iommu_release_dma_ownership(struct iommu_group *group)
		3200	{
		3201	int ret;
		3202
		3203	if (WARN_ON(!group->owner_cnt \|\| !group->owner \|\|
		3204	!xa_empty(&group->pasid_array)))
		3205	return;
3205		3206
3206	group->owner_cnt = 0;	3207	group->owner_cnt = 0;
3207	group->owner = NULL;	3208	group->owner = NULL;
3208	ret = __iommu_group_set_domain(group, group->default_domain);	3209	ret = __iommu_group_set_domain(group, group->default_domain);
3209	WARN(ret, "iommu driver failed to attach the default domain");	3210	WARN(ret, "iommu driver failed to attach the default domain");
		3211	}
3210		3212
3211	unlock_out:	3213	/**
		3214	* iommu_group_release_dma_owner() - Release DMA ownership of a group
		3215	* @dev: The device
		3216	*
		3217	* Release the DMA ownership claimed by iommu_group_claim_dma_owner().
		3218	*/
		3219	void iommu_group_release_dma_owner(struct iommu_group *group)
		3220	{
		3221	mutex_lock(&group->mutex);
		3222	__iommu_release_dma_ownership(group);
3212	mutex_unlock(&group->mutex);	3223	mutex_unlock(&group->mutex);
3213	}	3224	}
3214	EXPORT_SYMBOL_GPL(iommu_group_release_dma_owner);	3225	EXPORT_SYMBOL_GPL(iommu_group_release_dma_owner);
3215		3226
		3227	/**
		3228	* iommu_device_release_dma_owner() - Release DMA ownership of a device
		3229	* @group: The device.
		3230	*
		3231	* Release the DMA ownership claimed by iommu_device_claim_dma_owner().
		3232	*/
		3233	void iommu_device_release_dma_owner(struct device *dev)
		3234	{
		3235	struct iommu_group *group = iommu_group_get(dev);
		3236
		3237	mutex_lock(&group->mutex);
		3238	if (group->owner_cnt > 1)
		3239	group->owner_cnt--;
		3240	else
		3241	__iommu_release_dma_ownership(group);
		3242	mutex_unlock(&group->mutex);
		3243	iommu_group_put(group);
		3244	}
		3245	EXPORT_SYMBOL_GPL(iommu_device_release_dma_owner);
		3246
3216	/**	3247	/**
3217	* iommu_group_dma_owner_claimed() - Query group dma ownership status	3248	* iommu_group_dma_owner_claimed() - Query group dma ownership status
3218	* @group: The group.	3249	* @group: The group.
@@ -3231,3 +3262,150 @@ bool iommu_group_dma_owner_claimed(struct iommu_group *group) · 12 unmodified lines
3231	return user;	3262	return user;
3232	}	3263	}
3233	EXPORT_SYMBOL_GPL(iommu_group_dma_owner_claimed);	3264	EXPORT_SYMBOL_GPL(iommu_group_dma_owner_claimed);
		3265
		3266	static int __iommu_set_group_pasid(struct iommu_domain *domain,
		3267	struct iommu_group *group, ioasid_t pasid)
		3268	{
		3269	struct group_device *device;
		3270	int ret = 0;
		3271
		3272	list_for_each_entry(device, &group->devices, list) {
		3273	ret = domain->ops->set_dev_pasid(domain, device->dev, pasid);
		3274	if (ret)
		3275	break;
		3276	}
		3277
		3278	return ret;
		3279	}
		3280
		3281	static void __iommu_remove_group_pasid(struct iommu_group *group,
		3282	ioasid_t pasid)
		3283	{
		3284	struct group_device *device;
		3285	const struct iommu_ops *ops;
		3286
		3287	list_for_each_entry(device, &group->devices, list) {
		3288	ops = dev_iommu_ops(device->dev);
		3289	ops->remove_dev_pasid(device->dev, pasid);
		3290	}
		3291	}
		3292
		3293	/*
		3294	* iommu_attach_device_pasid() - Attach a domain to pasid of device
		3295	* @domain: the iommu domain.
		3296	* @dev: the attached device.
		3297	* @pasid: the pasid of the device.
		3298	*
		3299	* Return: 0 on success, or an error.
		3300	*/
		3301	int iommu_attach_device_pasid(struct iommu_domain *domain,
		3302	struct device *dev, ioasid_t pasid)
		3303	{
		3304	struct iommu_group *group;
		3305	void *curr;
		3306	int ret;
		3307
		3308	if (!domain->ops->set_dev_pasid)
		3309	return -EOPNOTSUPP;
		3310
		3311	group = iommu_group_get(dev);
		3312	if (!group)
		3313	return -ENODEV;
		3314
		3315	mutex_lock(&group->mutex);
		3316	curr = xa_cmpxchg(&group->pasid_array, pasid, NULL, domain, GFP_KERNEL);
		3317	if (curr) {
		3318	ret = xa_err(curr) ? : -EBUSY;
		3319	goto out_unlock;
		3320	}
		3321
		3322	ret = __iommu_set_group_pasid(domain, group, pasid);
		3323	if (ret) {
		3324	__iommu_remove_group_pasid(group, pasid);
		3325	xa_erase(&group->pasid_array, pasid);
		3326	}
		3327	out_unlock:
		3328	mutex_unlock(&group->mutex);
		3329	iommu_group_put(group);
		3330
		3331	return ret;
		3332	}
		3333	EXPORT_SYMBOL_GPL(iommu_attach_device_pasid);
		3334
		3335	/*
		3336	* iommu_detach_device_pasid() - Detach the domain from pasid of device
		3337	* @domain: the iommu domain.
		3338	* @dev: the attached device.
		3339	* @pasid: the pasid of the device.
		3340	*
		3341	* The @domain must have been attached to @pasid of the @dev with
		3342	* iommu_attach_device_pasid().
		3343	*/
		3344	void iommu_detach_device_pasid(struct iommu_domain domain, struct device dev,
		3345	ioasid_t pasid)
		3346	{
		3347	struct iommu_group *group = iommu_group_get(dev);
		3348
		3349	mutex_lock(&group->mutex);
		3350	__iommu_remove_group_pasid(group, pasid);
		3351	WARN_ON(xa_erase(&group->pasid_array, pasid) != domain);
		3352	mutex_unlock(&group->mutex);
		3353
		3354	iommu_group_put(group);
		3355	}
		3356	EXPORT_SYMBOL_GPL(iommu_detach_device_pasid);
		3357
		3358	/*
		3359	* iommu_get_domain_for_dev_pasid() - Retrieve domain for @pasid of @dev
		3360	* @dev: the queried device
		3361	* @pasid: the pasid of the device
		3362	* @type: matched domain type, 0 for any match
		3363	*
		3364	* This is a variant of iommu_get_domain_for_dev(). It returns the existing
		3365	* domain attached to pasid of a device. Callers must hold a lock around this
		3366	* function, and both iommu_attach/detach_dev_pasid() whenever a domain of
		3367	* type is being manipulated. This API does not internally resolve races with
		3368	* attach/detach.
		3369	*
		3370	* Return: attached domain on success, NULL otherwise.
		3371	*/
		3372	struct iommu_domain iommu_get_domain_for_dev_pasid(struct device dev,
		3373	ioasid_t pasid,
		3374	unsigned int type)
		3375	{
		3376	struct iommu_domain *domain;
		3377	struct iommu_group *group;
		3378
		3379	group = iommu_group_get(dev);
		3380	if (!group)
		3381	return NULL;
		3382
		3383	xa_lock(&group->pasid_array);
		3384	domain = xa_load(&group->pasid_array, pasid);
		3385	if (type && domain && domain->type != type)
		3386	domain = ERR_PTR(-EBUSY);
		3387	xa_unlock(&group->pasid_array);
		3388	iommu_group_put(group);
		3389
		3390	return domain;
		3391	}
		3392	EXPORT_SYMBOL_GPL(iommu_get_domain_for_dev_pasid);
		3393
		3394	struct iommu_domain iommu_sva_domain_alloc(struct device dev,
		3395	struct mm_struct *mm)
		3396	{
		3397	const struct iommu_ops *ops = dev_iommu_ops(dev);
		3398	struct iommu_domain *domain;
		3399
		3400	domain = ops->domain_alloc(IOMMU_DOMAIN_SVA);
		3401	if (!domain)
		3402	return NULL;
		3403
		3404	domain->type = IOMMU_DOMAIN_SVA;
		3405	mmgrab(mm);
		3406	domain->mm = mm;
		3407	domain->iopf_handler = iommu_sva_handle_iopf;
		3408	domain->fault_data = mm;
		3409
		3410	return domain;
		3411	}

drivers/iommu/iommufd/Kconfig+44−0

@@ -0,0 +1,44 @@
	1	# SPDX-License-Identifier: GPL-2.0-only
	2	config IOMMUFD
	3	tristate "IOMMU Userspace API"
	4	select INTERVAL_TREE
	5	select INTERVAL_TREE_SPAN_ITER
	6	select IOMMU_API
	7	default n
	8	help
	9	Provides /dev/iommu, the user API to control the IOMMU subsystem as
	10	it relates to managing IO page tables that point at user space memory.
	11
	12	If you don't know what to do here, say N.
	13
	14	if IOMMUFD
	15	config IOMMUFD_VFIO_CONTAINER
	16	bool "IOMMUFD provides the VFIO container /dev/vfio/vfio"
	17	depends on VFIO && !VFIO_CONTAINER
	18	default VFIO && !VFIO_CONTAINER
	19	help
	20	IOMMUFD will provide /dev/vfio/vfio instead of VFIO. This relies on
	21	IOMMUFD providing compatibility emulation to give the same ioctls.
	22	It provides an option to build a kernel with legacy VFIO components
	23	removed.
	24
	25	IOMMUFD VFIO container emulation is known to lack certain features
	26	of the native VFIO container, such as no-IOMMU support, peer-to-peer
	27	DMA mapping, PPC IOMMU support, as well as other potentially
	28	undiscovered gaps. This option is currently intended for the
	29	purpose of testing IOMMUFD with unmodified userspace supporting VFIO
	30	and making use of the Type1 VFIO IOMMU backend. General purpose
	31	enabling of this option is currently discouraged.
	32
	33	Unless testing IOMMUFD, say N here.
	34
	35	config IOMMUFD_TEST
	36	bool "IOMMU Userspace API Test support"
	37	depends on DEBUG_KERNEL
	38	depends on FAULT_INJECTION
	39	depends on RUNTIME_TESTING_MENU
	40	default n
	41	help
	42	This is dangerous, do not enable unless running
	43	tools/testing/selftests/iommu
	44	endif

drivers/iommu/iommufd/Makefile+13−0

@@ -0,0 +1,13 @@
	1	# SPDX-License-Identifier: GPL-2.0-only
	2	iommufd-y := \
	3	device.o \
	4	hw_pagetable.o \
	5	io_pagetable.o \
	6	ioas.o \
	7	main.o \
	8	pages.o \
	9	vfio_compat.o
	10
	11	iommufd-$(CONFIG_IOMMUFD_TEST) += selftest.o
	12
	13	obj-$(CONFIG_IOMMUFD) += iommufd.o

drivers/iommu/iommufd/device.c+772−0

@@ -0,0 +1,772 @@
	1	// SPDX-License-Identifier: GPL-2.0-only
	2	/* Copyright (c) 2021-2022, NVIDIA CORPORATION & AFFILIATES
	3	*/
	4	#include <linux/iommufd.h>
	5	#include <linux/slab.h>
	6	#include <linux/iommu.h>
	7	#include <linux/irqdomain.h>
	8
	9	#include "io_pagetable.h"
	10	#include "iommufd_private.h"
	11
	12	static bool allow_unsafe_interrupts;
	13	module_param(allow_unsafe_interrupts, bool, S_IRUGO \| S_IWUSR);
	14	MODULE_PARM_DESC(
	15	allow_unsafe_interrupts,
	16	"Allow IOMMUFD to bind to devices even if the platform cannot isolate "
	17	"the MSI interrupt window. Enabling this is a security weakness.");
	18
	19	/*
	20	* A iommufd_device object represents the binding relationship between a
	21	* consuming driver and the iommufd. These objects are created/destroyed by
	22	* external drivers, not by userspace.
	23	*/
	24	struct iommufd_device {
	25	struct iommufd_object obj;
	26	struct iommufd_ctx *ictx;
	27	struct iommufd_hw_pagetable *hwpt;
	28	/* Head at iommufd_hw_pagetable::devices */
	29	struct list_head devices_item;
	30	/* always the physical device */
	31	struct device *dev;
	32	struct iommu_group *group;
	33	bool enforce_cache_coherency;
	34	};
	35
	36	void iommufd_device_destroy(struct iommufd_object *obj)
	37	{
	38	struct iommufd_device *idev =
	39	container_of(obj, struct iommufd_device, obj);
	40
	41	iommu_device_release_dma_owner(idev->dev);
	42	iommu_group_put(idev->group);
	43	iommufd_ctx_put(idev->ictx);
	44	}
	45
	46	/**
	47	* iommufd_device_bind - Bind a physical device to an iommu fd
	48	* @ictx: iommufd file descriptor
	49	* @dev: Pointer to a physical device struct
	50	* @id: Output ID number to return to userspace for this device
	51	*
	52	* A successful bind establishes an ownership over the device and returns
	53	* struct iommufd_device pointer, otherwise returns error pointer.
	54	*
	55	* A driver using this API must set driver_managed_dma and must not touch
	56	* the device until this routine succeeds and establishes ownership.
	57	*
	58	* Binding a PCI device places the entire RID under iommufd control.
	59	*
	60	* The caller must undo this with iommufd_device_unbind()
	61	*/
	62	struct iommufd_device iommufd_device_bind(struct iommufd_ctx ictx,
	63	struct device dev, u32 id)
	64	{
	65	struct iommufd_device *idev;
	66	struct iommu_group *group;
	67	int rc;
	68
	69	/*
	70	* iommufd always sets IOMMU_CACHE because we offer no way for userspace
	71	* to restore cache coherency.
	72	*/
	73	if (!device_iommu_capable(dev, IOMMU_CAP_CACHE_COHERENCY))
	74	return ERR_PTR(-EINVAL);
	75
	76	group = iommu_group_get(dev);
	77	if (!group)
	78	return ERR_PTR(-ENODEV);
	79
	80	rc = iommu_device_claim_dma_owner(dev, ictx);
	81	if (rc)
	82	goto out_group_put;
	83
	84	idev = iommufd_object_alloc(ictx, idev, IOMMUFD_OBJ_DEVICE);
	85	if (IS_ERR(idev)) {
	86	rc = PTR_ERR(idev);
	87	goto out_release_owner;
	88	}
	89	idev->ictx = ictx;
	90	iommufd_ctx_get(ictx);
	91	idev->dev = dev;
	92	idev->enforce_cache_coherency =
	93	device_iommu_capable(dev, IOMMU_CAP_ENFORCE_CACHE_COHERENCY);
	94	/* The calling driver is a user until iommufd_device_unbind() */
	95	refcount_inc(&idev->obj.users);
	96	/* group refcount moves into iommufd_device */
	97	idev->group = group;
	98
	99	/*
	100	* If the caller fails after this success it must call
	101	* iommufd_unbind_device() which is safe since we hold this refcount.
	102	* This also means the device is a leaf in the graph and no other object
	103	* can take a reference on it.
	104	*/
	105	iommufd_object_finalize(ictx, &idev->obj);
	106	*id = idev->obj.id;
	107	return idev;
	108
	109	out_release_owner:
	110	iommu_device_release_dma_owner(dev);
	111	out_group_put:
	112	iommu_group_put(group);
	113	return ERR_PTR(rc);
	114	}
	115	EXPORT_SYMBOL_NS_GPL(iommufd_device_bind, IOMMUFD);
	116
	117	/**
	118	* iommufd_device_unbind - Undo iommufd_device_bind()
	119	* @idev: Device returned by iommufd_device_bind()
	120	*
	121	* Release the device from iommufd control. The DMA ownership will return back
	122	* to unowned with DMA controlled by the DMA API. This invalidates the
	123	* iommufd_device pointer, other APIs that consume it must not be called
	124	* concurrently.
	125	*/
	126	void iommufd_device_unbind(struct iommufd_device *idev)
	127	{
	128	bool was_destroyed;
	129
	130	was_destroyed = iommufd_object_destroy_user(idev->ictx, &idev->obj);
	131	WARN_ON(!was_destroyed);
	132	}
	133	EXPORT_SYMBOL_NS_GPL(iommufd_device_unbind, IOMMUFD);
	134
	135	static int iommufd_device_setup_msi(struct iommufd_device *idev,
	136	struct iommufd_hw_pagetable *hwpt,
	137	phys_addr_t sw_msi_start)
	138	{
	139	int rc;
	140
	141	/*
	142	* If the IOMMU driver gives a IOMMU_RESV_SW_MSI then it is asking us to
	143	* call iommu_get_msi_cookie() on its behalf. This is necessary to setup
	144	* the MSI window so iommu_dma_prepare_msi() can install pages into our
	145	* domain after request_irq(). If it is not done interrupts will not
	146	* work on this domain.
	147	*
	148	* FIXME: This is conceptually broken for iommufd since we want to allow
	149	* userspace to change the domains, eg switch from an identity IOAS to a
	150	* DMA IOAS. There is currently no way to create a MSI window that
	151	* matches what the IRQ layer actually expects in a newly created
	152	* domain.
	153	*/
	154	if (sw_msi_start != PHYS_ADDR_MAX && !hwpt->msi_cookie) {
	155	rc = iommu_get_msi_cookie(hwpt->domain, sw_msi_start);
	156	if (rc)
	157	return rc;
	158
	159	/*
	160	* iommu_get_msi_cookie() can only be called once per domain,
	161	* it returns -EBUSY on later calls.
	162	*/
	163	hwpt->msi_cookie = true;
	164	}
	165
	166	/*
	167	* For historical compat with VFIO the insecure interrupt path is
	168	* allowed if the module parameter is set. Insecure means that a MemWr
	169	* operation from the device (eg a simple DMA) cannot trigger an
	170	* interrupt outside this iommufd context.
	171	*/
	172	if (!device_iommu_capable(idev->dev, IOMMU_CAP_INTR_REMAP) &&
	173	!irq_domain_check_msi_remap()) {
	174	if (!allow_unsafe_interrupts)
	175	return -EPERM;
	176
	177	dev_warn(
	178	idev->dev,
	179	"MSI interrupts are not secure, they cannot be isolated by the platform. "
	180	"Check that platform features like interrupt remapping are enabled. "
	181	"Use the \"allow_unsafe_interrupts\" module parameter to override\n");
	182	}
	183	return 0;
	184	}
	185
	186	static bool iommufd_hw_pagetable_has_group(struct iommufd_hw_pagetable *hwpt,
	187	struct iommu_group *group)
	188	{
	189	struct iommufd_device *cur_dev;
	190
	191	list_for_each_entry(cur_dev, &hwpt->devices, devices_item)
	192	if (cur_dev->group == group)
	193	return true;
	194	return false;
	195	}
	196
	197	static int iommufd_device_do_attach(struct iommufd_device *idev,
	198	struct iommufd_hw_pagetable *hwpt)
	199	{
	200	phys_addr_t sw_msi_start = PHYS_ADDR_MAX;
	201	int rc;
	202
	203	mutex_lock(&hwpt->devices_lock);
	204
	205	/*
	206	* Try to upgrade the domain we have, it is an iommu driver bug to
	207	* report IOMMU_CAP_ENFORCE_CACHE_COHERENCY but fail
	208	* enforce_cache_coherency when there are no devices attached to the
	209	* domain.
	210	*/
	211	if (idev->enforce_cache_coherency && !hwpt->enforce_cache_coherency) {
	212	if (hwpt->domain->ops->enforce_cache_coherency)
	213	hwpt->enforce_cache_coherency =
	214	hwpt->domain->ops->enforce_cache_coherency(
	215	hwpt->domain);
	216	if (!hwpt->enforce_cache_coherency) {
	217	WARN_ON(list_empty(&hwpt->devices));
	218	rc = -EINVAL;
	219	goto out_unlock;
	220	}
	221	}
	222
	223	rc = iopt_table_enforce_group_resv_regions(&hwpt->ioas->iopt, idev->dev,
	224	idev->group, &sw_msi_start);
	225	if (rc)
	226	goto out_unlock;
	227
	228	rc = iommufd_device_setup_msi(idev, hwpt, sw_msi_start);
	229	if (rc)
	230	goto out_iova;
	231
	232	/*
	233	* FIXME: Hack around missing a device-centric iommu api, only attach to
	234	* the group once for the first device that is in the group.
	235	*/
	236	if (!iommufd_hw_pagetable_has_group(hwpt, idev->group)) {
	237	rc = iommu_attach_group(hwpt->domain, idev->group);
	238	if (rc)
	239	goto out_iova;
	240
	241	if (list_empty(&hwpt->devices)) {
	242	rc = iopt_table_add_domain(&hwpt->ioas->iopt,
	243	hwpt->domain);
	244	if (rc)
	245	goto out_detach;
	246	}
	247	}
	248
	249	idev->hwpt = hwpt;
	250	refcount_inc(&hwpt->obj.users);
	251	list_add(&idev->devices_item, &hwpt->devices);
	252	mutex_unlock(&hwpt->devices_lock);
	253	return 0;
	254
	255	out_detach:
	256	iommu_detach_group(hwpt->domain, idev->group);
	257	out_iova:
	258	iopt_remove_reserved_iova(&hwpt->ioas->iopt, idev->dev);
	259	out_unlock:
	260	mutex_unlock(&hwpt->devices_lock);
	261	return rc;
	262	}
	263
	264	/*
	265	* When automatically managing the domains we search for a compatible domain in
	266	* the iopt and if one is found use it, otherwise create a new domain.
	267	* Automatic domain selection will never pick a manually created domain.
	268	*/
	269	static int iommufd_device_auto_get_domain(struct iommufd_device *idev,
	270	struct iommufd_ioas *ioas)
	271	{
	272	struct iommufd_hw_pagetable *hwpt;
	273	int rc;
	274
	275	/*
	276	* There is no differentiation when domains are allocated, so any domain
	277	* that is willing to attach to the device is interchangeable with any
	278	* other.
	279	*/
	280	mutex_lock(&ioas->mutex);
	281	list_for_each_entry(hwpt, &ioas->hwpt_list, hwpt_item) {
	282	if (!hwpt->auto_domain)
	283	continue;
	284
	285	rc = iommufd_device_do_attach(idev, hwpt);
	286
	287	/*
	288	* -EINVAL means the domain is incompatible with the device.
	289	* Other error codes should propagate to userspace as failure.
	290	* Success means the domain is attached.
	291	*/
	292	if (rc == -EINVAL)
	293	continue;
	294	goto out_unlock;
	295	}
	296
	297	hwpt = iommufd_hw_pagetable_alloc(idev->ictx, ioas, idev->dev);
	298	if (IS_ERR(hwpt)) {
	299	rc = PTR_ERR(hwpt);
	300	goto out_unlock;
	301	}
	302	hwpt->auto_domain = true;
	303
	304	rc = iommufd_device_do_attach(idev, hwpt);
	305	if (rc)
	306	goto out_abort;
	307	list_add_tail(&hwpt->hwpt_item, &ioas->hwpt_list);
	308
	309	mutex_unlock(&ioas->mutex);
	310	iommufd_object_finalize(idev->ictx, &hwpt->obj);
	311	return 0;
	312
	313	out_abort:
	314	iommufd_object_abort_and_destroy(idev->ictx, &hwpt->obj);
	315	out_unlock:
	316	mutex_unlock(&ioas->mutex);
	317	return rc;
	318	}
	319
	320	/**
	321	* iommufd_device_attach - Connect a device from an iommu_domain
	322	* @idev: device to attach
	323	* @pt_id: Input a IOMMUFD_OBJ_IOAS, or IOMMUFD_OBJ_HW_PAGETABLE
	324	* Output the IOMMUFD_OBJ_HW_PAGETABLE ID
	325	*
	326	* This connects the device to an iommu_domain, either automatically or manually
	327	* selected. Once this completes the device could do DMA.
	328	*
	329	* The caller should return the resulting pt_id back to userspace.
	330	* This function is undone by calling iommufd_device_detach().
	331	*/
	332	int iommufd_device_attach(struct iommufd_device idev, u32 pt_id)
	333	{
	334	struct iommufd_object *pt_obj;
	335	int rc;
	336
	337	pt_obj = iommufd_get_object(idev->ictx, *pt_id, IOMMUFD_OBJ_ANY);
	338	if (IS_ERR(pt_obj))
	339	return PTR_ERR(pt_obj);
	340
	341	switch (pt_obj->type) {
	342	case IOMMUFD_OBJ_HW_PAGETABLE: {
	343	struct iommufd_hw_pagetable *hwpt =
	344	container_of(pt_obj, struct iommufd_hw_pagetable, obj);
	345
	346	rc = iommufd_device_do_attach(idev, hwpt);
	347	if (rc)
	348	goto out_put_pt_obj;
	349
	350	mutex_lock(&hwpt->ioas->mutex);
	351	list_add_tail(&hwpt->hwpt_item, &hwpt->ioas->hwpt_list);
	352	mutex_unlock(&hwpt->ioas->mutex);
	353	break;
	354	}
	355	case IOMMUFD_OBJ_IOAS: {
	356	struct iommufd_ioas *ioas =
	357	container_of(pt_obj, struct iommufd_ioas, obj);
	358
	359	rc = iommufd_device_auto_get_domain(idev, ioas);
	360	if (rc)
	361	goto out_put_pt_obj;
	362	break;
	363	}
	364	default:
	365	rc = -EINVAL;
	366	goto out_put_pt_obj;
	367	}
	368
	369	refcount_inc(&idev->obj.users);
	370	*pt_id = idev->hwpt->obj.id;
	371	rc = 0;
	372
	373	out_put_pt_obj:
	374	iommufd_put_object(pt_obj);
	375	return rc;
	376	}
	377	EXPORT_SYMBOL_NS_GPL(iommufd_device_attach, IOMMUFD);
	378
	379	/**
	380	* iommufd_device_detach - Disconnect a device to an iommu_domain
	381	* @idev: device to detach
	382	*
	383	* Undo iommufd_device_attach(). This disconnects the idev from the previously
	384	* attached pt_id. The device returns back to a blocked DMA translation.
	385	*/
	386	void iommufd_device_detach(struct iommufd_device *idev)
	387	{
	388	struct iommufd_hw_pagetable *hwpt = idev->hwpt;
	389
	390	mutex_lock(&hwpt->ioas->mutex);
	391	mutex_lock(&hwpt->devices_lock);
	392	list_del(&idev->devices_item);
	393	if (!iommufd_hw_pagetable_has_group(hwpt, idev->group)) {
	394	if (list_empty(&hwpt->devices)) {
	395	iopt_table_remove_domain(&hwpt->ioas->iopt,
	396	hwpt->domain);
	397	list_del(&hwpt->hwpt_item);
	398	}
	399	iommu_detach_group(hwpt->domain, idev->group);
	400	}
	401	iopt_remove_reserved_iova(&hwpt->ioas->iopt, idev->dev);
	402	mutex_unlock(&hwpt->devices_lock);
	403	mutex_unlock(&hwpt->ioas->mutex);
	404
	405	if (hwpt->auto_domain)
	406	iommufd_object_destroy_user(idev->ictx, &hwpt->obj);
	407	else
	408	refcount_dec(&hwpt->obj.users);
	409
	410	idev->hwpt = NULL;
	411
	412	refcount_dec(&idev->obj.users);
	413	}
	414	EXPORT_SYMBOL_NS_GPL(iommufd_device_detach, IOMMUFD);
	415
	416	void iommufd_access_destroy_object(struct iommufd_object *obj)
	417	{
	418	struct iommufd_access *access =
	419	container_of(obj, struct iommufd_access, obj);
	420
	421	iopt_remove_access(&access->ioas->iopt, access);
	422	iommufd_ctx_put(access->ictx);
	423	refcount_dec(&access->ioas->obj.users);
	424	}
	425
	426	/**
	427	* iommufd_access_create - Create an iommufd_access
	428	* @ictx: iommufd file descriptor
	429	* @ioas_id: ID for a IOMMUFD_OBJ_IOAS
	430	* @ops: Driver's ops to associate with the access
	431	* @data: Opaque data to pass into ops functions
	432	*
	433	* An iommufd_access allows a driver to read/write to the IOAS without using
	434	* DMA. The underlying CPU memory can be accessed using the
	435	* iommufd_access_pin_pages() or iommufd_access_rw() functions.
	436	*
	437	* The provided ops are required to use iommufd_access_pin_pages().
	438	*/
	439	struct iommufd_access *
	440	iommufd_access_create(struct iommufd_ctx *ictx, u32 ioas_id,
	441	const struct iommufd_access_ops ops, void data)
	442	{
	443	struct iommufd_access *access;
	444	struct iommufd_object *obj;
	445	int rc;
	446
	447	/*
	448	* There is no uAPI for the access object, but to keep things symmetric
	449	* use the object infrastructure anyhow.
	450	*/
	451	access = iommufd_object_alloc(ictx, access, IOMMUFD_OBJ_ACCESS);
	452	if (IS_ERR(access))
	453	return access;
	454
	455	access->data = data;
	456	access->ops = ops;
	457
	458	obj = iommufd_get_object(ictx, ioas_id, IOMMUFD_OBJ_IOAS);
	459	if (IS_ERR(obj)) {
	460	rc = PTR_ERR(obj);
	461	goto out_abort;
	462	}
	463	access->ioas = container_of(obj, struct iommufd_ioas, obj);
	464	iommufd_ref_to_users(obj);
	465
	466	if (ops->needs_pin_pages)
	467	access->iova_alignment = PAGE_SIZE;
	468	else
	469	access->iova_alignment = 1;
	470	rc = iopt_add_access(&access->ioas->iopt, access);
	471	if (rc)
	472	goto out_put_ioas;
	473
	474	/* The calling driver is a user until iommufd_access_destroy() */
	475	refcount_inc(&access->obj.users);
	476	access->ictx = ictx;
	477	iommufd_ctx_get(ictx);
	478	iommufd_object_finalize(ictx, &access->obj);
	479	return access;
	480	out_put_ioas:
	481	refcount_dec(&access->ioas->obj.users);
	482	out_abort:
	483	iommufd_object_abort(ictx, &access->obj);
	484	return ERR_PTR(rc);
	485	}
	486	EXPORT_SYMBOL_NS_GPL(iommufd_access_create, IOMMUFD);
	487
	488	/**
	489	* iommufd_access_destroy - Destroy an iommufd_access
	490	* @access: The access to destroy
	491	*
	492	* The caller must stop using the access before destroying it.
	493	*/
	494	void iommufd_access_destroy(struct iommufd_access *access)
	495	{
	496	bool was_destroyed;
	497
	498	was_destroyed = iommufd_object_destroy_user(access->ictx, &access->obj);
	499	WARN_ON(!was_destroyed);
	500	}
	501	EXPORT_SYMBOL_NS_GPL(iommufd_access_destroy, IOMMUFD);
	502
	503	/**
	504	* iommufd_access_notify_unmap - Notify users of an iopt to stop using it
	505	* @iopt: iopt to work on
	506	* @iova: Starting iova in the iopt
	507	* @length: Number of bytes
	508	*
	509	* After this function returns there should be no users attached to the pages
	510	* linked to this iopt that intersect with iova,length. Anyone that has attached
	511	* a user through iopt_access_pages() needs to detach it through
	512	* iommufd_access_unpin_pages() before this function returns.
	513	*
	514	* iommufd_access_destroy() will wait for any outstanding unmap callback to
	515	* complete. Once iommufd_access_destroy() no unmap ops are running or will
	516	* run in the future. Due to this a driver must not create locking that prevents
	517	* unmap to complete while iommufd_access_destroy() is running.
	518	*/
	519	void iommufd_access_notify_unmap(struct io_pagetable *iopt, unsigned long iova,
	520	unsigned long length)
	521	{
	522	struct iommufd_ioas *ioas =
	523	container_of(iopt, struct iommufd_ioas, iopt);
	524	struct iommufd_access *access;
	525	unsigned long index;
	526
	527	xa_lock(&ioas->iopt.access_list);
	528	xa_for_each(&ioas->iopt.access_list, index, access) {
	529	if (!iommufd_lock_obj(&access->obj))
	530	continue;
	531	xa_unlock(&ioas->iopt.access_list);
	532
	533	access->ops->unmap(access->data, iova, length);
	534
	535	iommufd_put_object(&access->obj);
	536	xa_lock(&ioas->iopt.access_list);
	537	}
	538	xa_unlock(&ioas->iopt.access_list);
	539	}
	540
	541	/**
	542	* iommufd_access_unpin_pages() - Undo iommufd_access_pin_pages
	543	* @access: IOAS access to act on
	544	* @iova: Starting IOVA
	545	* @length: Number of bytes to access
	546	*
	547	* Return the struct page's. The caller must stop accessing them before calling
	548	* this. The iova/length must exactly match the one provided to access_pages.
	549	*/
	550	void iommufd_access_unpin_pages(struct iommufd_access *access,
	551	unsigned long iova, unsigned long length)
	552	{
	553	struct io_pagetable *iopt = &access->ioas->iopt;
	554	struct iopt_area_contig_iter iter;
	555	unsigned long last_iova;
	556	struct iopt_area *area;
	557
	558	if (WARN_ON(!length) \|\|
	559	WARN_ON(check_add_overflow(iova, length - 1, &last_iova)))
	560	return;
	561
	562	down_read(&iopt->iova_rwsem);
	563	iopt_for_each_contig_area(&iter, area, iopt, iova, last_iova)
	564	iopt_area_remove_access(
	565	area, iopt_area_iova_to_index(area, iter.cur_iova),
	566	iopt_area_iova_to_index(
	567	area,
	568	min(last_iova, iopt_area_last_iova(area))));
	569	up_read(&iopt->iova_rwsem);
	570	WARN_ON(!iopt_area_contig_done(&iter));
	571	}
	572	EXPORT_SYMBOL_NS_GPL(iommufd_access_unpin_pages, IOMMUFD);
	573
	574	static bool iopt_area_contig_is_aligned(struct iopt_area_contig_iter *iter)
	575	{
	576	if (iopt_area_start_byte(iter->area, iter->cur_iova) % PAGE_SIZE)
	577	return false;
	578
	579	if (!iopt_area_contig_done(iter) &&
	580	(iopt_area_start_byte(iter->area, iopt_area_last_iova(iter->area)) %
	581	PAGE_SIZE) != (PAGE_SIZE - 1))
	582	return false;
	583	return true;
	584	}
	585
	586	static bool check_area_prot(struct iopt_area *area, unsigned int flags)
	587	{
	588	if (flags & IOMMUFD_ACCESS_RW_WRITE)
	589	return area->iommu_prot & IOMMU_WRITE;
	590	return area->iommu_prot & IOMMU_READ;
	591	}
	592
	593	/**
	594	* iommufd_access_pin_pages() - Return a list of pages under the iova
	595	* @access: IOAS access to act on
	596	* @iova: Starting IOVA
	597	* @length: Number of bytes to access
	598	* @out_pages: Output page list
	599	* @flags: IOPMMUFD_ACCESS_RW_* flags
	600	*
	601	* Reads @length bytes starting at iova and returns the struct page * pointers.
	602	* These can be kmap'd by the caller for CPU access.
	603	*
	604	* The caller must perform iommufd_access_unpin_pages() when done to balance
	605	* this.
	606	*
	607	* This API always requires a page aligned iova. This happens naturally if the
	608	* ioas alignment is >= PAGE_SIZE and the iova is PAGE_SIZE aligned. However
	609	* smaller alignments have corner cases where this API can fail on otherwise
	610	* aligned iova.
	611	*/
	612	int iommufd_access_pin_pages(struct iommufd_access *access, unsigned long iova,
	613	unsigned long length, struct page **out_pages,
	614	unsigned int flags)
	615	{
	616	struct io_pagetable *iopt = &access->ioas->iopt;
	617	struct iopt_area_contig_iter iter;
	618	unsigned long last_iova;
	619	struct iopt_area *area;
	620	int rc;
	621
	622	/* Driver's ops don't support pin_pages */
	623	if (IS_ENABLED(CONFIG_IOMMUFD_TEST) &&
	624	WARN_ON(access->iova_alignment != PAGE_SIZE \|\| !access->ops->unmap))
	625	return -EINVAL;
	626
	627	if (!length)
	628	return -EINVAL;
	629	if (check_add_overflow(iova, length - 1, &last_iova))
	630	return -EOVERFLOW;
	631
	632	down_read(&iopt->iova_rwsem);
	633	iopt_for_each_contig_area(&iter, area, iopt, iova, last_iova) {
	634	unsigned long last = min(last_iova, iopt_area_last_iova(area));
	635	unsigned long last_index = iopt_area_iova_to_index(area, last);
	636	unsigned long index =
	637	iopt_area_iova_to_index(area, iter.cur_iova);
	638
	639	if (area->prevent_access \|\|
	640	!iopt_area_contig_is_aligned(&iter)) {
	641	rc = -EINVAL;
	642	goto err_remove;
	643	}
	644
	645	if (!check_area_prot(area, flags)) {
	646	rc = -EPERM;
	647	goto err_remove;
	648	}
	649
	650	rc = iopt_area_add_access(area, index, last_index, out_pages,
	651	flags);
	652	if (rc)
	653	goto err_remove;
	654	out_pages += last_index - index + 1;
	655	}
	656	if (!iopt_area_contig_done(&iter)) {
	657	rc = -ENOENT;
	658	goto err_remove;
	659	}
	660
	661	up_read(&iopt->iova_rwsem);
	662	return 0;
	663
	664	err_remove:
	665	if (iova < iter.cur_iova) {
	666	last_iova = iter.cur_iova - 1;
	667	iopt_for_each_contig_area(&iter, area, iopt, iova, last_iova)
	668	iopt_area_remove_access(
	669	area,
	670	iopt_area_iova_to_index(area, iter.cur_iova),
	671	iopt_area_iova_to_index(
	672	area, min(last_iova,
	673	iopt_area_last_iova(area))));
	674	}
	675	up_read(&iopt->iova_rwsem);
	676	return rc;
	677	}
	678	EXPORT_SYMBOL_NS_GPL(iommufd_access_pin_pages, IOMMUFD);
	679
	680	/**
	681	* iommufd_access_rw - Read or write data under the iova
	682	* @access: IOAS access to act on
	683	* @iova: Starting IOVA
	684	* @data: Kernel buffer to copy to/from
	685	* @length: Number of bytes to access
	686	* @flags: IOMMUFD_ACCESS_RW_* flags
	687	*
	688	* Copy kernel to/from data into the range given by IOVA/length. If flags
	689	* indicates IOMMUFD_ACCESS_RW_KTHREAD then a large copy can be optimized
	690	* by changing it into copy_to/from_user().
	691	*/
	692	int iommufd_access_rw(struct iommufd_access *access, unsigned long iova,
	693	void *data, size_t length, unsigned int flags)
	694	{
	695	struct io_pagetable *iopt = &access->ioas->iopt;
	696	struct iopt_area_contig_iter iter;
	697	struct iopt_area *area;
	698	unsigned long last_iova;
	699	int rc;
	700
	701	if (!length)
	702	return -EINVAL;
	703	if (check_add_overflow(iova, length - 1, &last_iova))
	704	return -EOVERFLOW;
	705
	706	down_read(&iopt->iova_rwsem);
	707	iopt_for_each_contig_area(&iter, area, iopt, iova, last_iova) {
	708	unsigned long last = min(last_iova, iopt_area_last_iova(area));
	709	unsigned long bytes = (last - iter.cur_iova) + 1;
	710
	711	if (area->prevent_access) {
	712	rc = -EINVAL;
	713	goto err_out;
	714	}
	715
	716	if (!check_area_prot(area, flags)) {
	717	rc = -EPERM;
	718	goto err_out;
	719	}
	720
	721	rc = iopt_pages_rw_access(
	722	area->pages, iopt_area_start_byte(area, iter.cur_iova),
	723	data, bytes, flags);
	724	if (rc)
	725	goto err_out;
	726	data += bytes;
	727	}
	728	if (!iopt_area_contig_done(&iter))
	729	rc = -ENOENT;
	730	err_out:
	731	up_read(&iopt->iova_rwsem);
	732	return rc;
	733	}
	734	EXPORT_SYMBOL_NS_GPL(iommufd_access_rw, IOMMUFD);
	735
	736	#ifdef CONFIG_IOMMUFD_TEST
	737	/*
	738	* Creating a real iommufd_device is too hard, bypass creating a iommufd_device
	739	* and go directly to attaching a domain.
	740	*/
	741	struct iommufd_hw_pagetable *
	742	iommufd_device_selftest_attach(struct iommufd_ctx *ictx,
	743	struct iommufd_ioas *ioas,
	744	struct device *mock_dev)
	745	{
	746	struct iommufd_hw_pagetable *hwpt;
	747	int rc;
	748
	749	hwpt = iommufd_hw_pagetable_alloc(ictx, ioas, mock_dev);
	750	if (IS_ERR(hwpt))
	751	return hwpt;
	752
	753	rc = iopt_table_add_domain(&hwpt->ioas->iopt, hwpt->domain);
	754	if (rc)
	755	goto out_hwpt;
	756
	757	refcount_inc(&hwpt->obj.users);
	758	iommufd_object_finalize(ictx, &hwpt->obj);
	759	return hwpt;
	760
	761	out_hwpt:
	762	iommufd_object_abort_and_destroy(ictx, &hwpt->obj);
	763	return ERR_PTR(rc);
	764	}
	765
	766	void iommufd_device_selftest_detach(struct iommufd_ctx *ictx,
	767	struct iommufd_hw_pagetable *hwpt)
	768	{
	769	iopt_table_remove_domain(&hwpt->ioas->iopt, hwpt->domain);
	770	refcount_dec(&hwpt->obj.users);
	771	}
	772	#endif

drivers/iommu/iommufd/double_span.h+53−0

@@ -0,0 +1,53 @@
	1	/* SPDX-License-Identifier: GPL-2.0 */
	2	/* Copyright (c) 2022, NVIDIA CORPORATION & AFFILIATES.
	3	*/
	4	#ifndef __IOMMUFD_DOUBLE_SPAN_H
	5	#define __IOMMUFD_DOUBLE_SPAN_H
	6
	7	#include <linux/interval_tree.h>
	8
	9	/*
	10	* This is a variation of the general interval_tree_span_iter that computes the
	11	* spans over the union of two different interval trees. Used ranges are broken
	12	* up and reported based on the tree that provides the interval. The first span
	13	* always takes priority. Like interval_tree_span_iter it is greedy and the same
	14	* value of is_used will not repeat on two iteration cycles.
	15	*/
	16	struct interval_tree_double_span_iter {
	17	struct rb_root_cached *itrees[2];
	18	struct interval_tree_span_iter spans[2];
	19	union {
	20	unsigned long start_hole;
	21	unsigned long start_used;
	22	};
	23	union {
	24	unsigned long last_hole;
	25	unsigned long last_used;
	26	};
	27	/* 0 = hole, 1 = used span[0], 2 = used span[1], -1 done iteration */
	28	int is_used;
	29	};
	30
	31	void interval_tree_double_span_iter_update(
	32	struct interval_tree_double_span_iter *iter);
	33	void interval_tree_double_span_iter_first(
	34	struct interval_tree_double_span_iter *iter,
	35	struct rb_root_cached itree1, struct rb_root_cached itree2,
	36	unsigned long first_index, unsigned long last_index);
	37	void interval_tree_double_span_iter_next(
	38	struct interval_tree_double_span_iter *iter);
	39
	40	static inline bool
	41	interval_tree_double_span_iter_done(struct interval_tree_double_span_iter *state)
	42	{
	43	return state->is_used == -1;
	44	}
	45
	46	#define interval_tree_for_each_double_span(span, itree1, itree2, first_index, \
	47	last_index) \
	48	for (interval_tree_double_span_iter_first(span, itree1, itree2, \
	49	first_index, last_index); \
	50	!interval_tree_double_span_iter_done(span); \
	51	interval_tree_double_span_iter_next(span))
	52
	53	#endif

drivers/iommu/iommufd/hw_pagetable.c+57−0

@@ -0,0 +1,57 @@
	1	// SPDX-License-Identifier: GPL-2.0-only
	2	/*
	3	* Copyright (c) 2021-2022, NVIDIA CORPORATION & AFFILIATES
	4	*/
	5	#include <linux/iommu.h>
	6
	7	#include "iommufd_private.h"
	8
	9	void iommufd_hw_pagetable_destroy(struct iommufd_object *obj)
	10	{
	11	struct iommufd_hw_pagetable *hwpt =
	12	container_of(obj, struct iommufd_hw_pagetable, obj);
	13
	14	WARN_ON(!list_empty(&hwpt->devices));
	15
	16	iommu_domain_free(hwpt->domain);
	17	refcount_dec(&hwpt->ioas->obj.users);
	18	mutex_destroy(&hwpt->devices_lock);
	19	}
	20
	21	/**
	22	* iommufd_hw_pagetable_alloc() - Get an iommu_domain for a device
	23	* @ictx: iommufd context
	24	* @ioas: IOAS to associate the domain with
	25	* @dev: Device to get an iommu_domain for
	26	*
	27	* Allocate a new iommu_domain and return it as a hw_pagetable.
	28	*/
	29	struct iommufd_hw_pagetable *
	30	iommufd_hw_pagetable_alloc(struct iommufd_ctx ictx, struct iommufd_ioas ioas,
	31	struct device *dev)
	32	{
	33	struct iommufd_hw_pagetable *hwpt;
	34	int rc;
	35
	36	hwpt = iommufd_object_alloc(ictx, hwpt, IOMMUFD_OBJ_HW_PAGETABLE);
	37	if (IS_ERR(hwpt))
	38	return hwpt;
	39
	40	hwpt->domain = iommu_domain_alloc(dev->bus);
	41	if (!hwpt->domain) {
	42	rc = -ENOMEM;
	43	goto out_abort;
	44	}
	45
	46	INIT_LIST_HEAD(&hwpt->devices);
	47	INIT_LIST_HEAD(&hwpt->hwpt_item);
	48	mutex_init(&hwpt->devices_lock);
	49	/* Pairs with iommufd_hw_pagetable_destroy() */
	50	refcount_inc(&ioas->obj.users);
	51	hwpt->ioas = ioas;
	52	return hwpt;
	53
	54	out_abort:
	55	iommufd_object_abort(ictx, &hwpt->obj);
	56	return ERR_PTR(rc);
	57	}

drivers/iommu/iommufd/io_pagetable.c+1216−0

@@ -0,0 +1,1216 @@
	1	// SPDX-License-Identifier: GPL-2.0
	2	/* Copyright (c) 2021-2022, NVIDIA CORPORATION & AFFILIATES.
	3	*
	4	* The io_pagetable is the top of datastructure that maps IOVA's to PFNs. The
	5	* PFNs can be placed into an iommu_domain, or returned to the caller as a page
	6	* list for access by an in-kernel user.
	7	*
	8	* The datastructure uses the iopt_pages to optimize the storage of the PFNs
	9	* between the domains and xarray.
	10	*/
	11	#include <linux/iommufd.h>
	12	#include <linux/lockdep.h>
	13	#include <linux/iommu.h>
	14	#include <linux/sched/mm.h>
	15	#include <linux/err.h>
	16	#include <linux/slab.h>
	17	#include <linux/errno.h>
	18
	19	#include "io_pagetable.h"
	20	#include "double_span.h"
	21
	22	struct iopt_pages_list {
	23	struct iopt_pages *pages;
	24	struct iopt_area *area;
	25	struct list_head next;
	26	unsigned long start_byte;
	27	unsigned long length;
	28	};
	29
	30	struct iopt_area iopt_area_contig_init(struct iopt_area_contig_iter iter,
	31	struct io_pagetable *iopt,
	32	unsigned long iova,
	33	unsigned long last_iova)
	34	{
	35	lockdep_assert_held(&iopt->iova_rwsem);
	36
	37	iter->cur_iova = iova;
	38	iter->last_iova = last_iova;
	39	iter->area = iopt_area_iter_first(iopt, iova, iova);
	40	if (!iter->area)
	41	return NULL;
	42	if (!iter->area->pages) {
	43	iter->area = NULL;
	44	return NULL;
	45	}
	46	return iter->area;
	47	}
	48
	49	struct iopt_area iopt_area_contig_next(struct iopt_area_contig_iter iter)
	50	{
	51	unsigned long last_iova;
	52
	53	if (!iter->area)
	54	return NULL;
	55	last_iova = iopt_area_last_iova(iter->area);
	56	if (iter->last_iova <= last_iova)
	57	return NULL;
	58
	59	iter->cur_iova = last_iova + 1;
	60	iter->area = iopt_area_iter_next(iter->area, iter->cur_iova,
	61	iter->last_iova);
	62	if (!iter->area)
	63	return NULL;
	64	if (iter->cur_iova != iopt_area_iova(iter->area) \|\|
	65	!iter->area->pages) {
	66	iter->area = NULL;
	67	return NULL;
	68	}
	69	return iter->area;
	70	}
	71
	72	static bool __alloc_iova_check_hole(struct interval_tree_double_span_iter *span,
	73	unsigned long length,
	74	unsigned long iova_alignment,
	75	unsigned long page_offset)
	76	{
	77	if (span->is_used \|\| span->last_hole - span->start_hole < length - 1)
	78	return false;
	79
	80	span->start_hole = ALIGN(span->start_hole, iova_alignment) \|
	81	page_offset;
	82	if (span->start_hole > span->last_hole \|\|
	83	span->last_hole - span->start_hole < length - 1)
	84	return false;
	85	return true;
	86	}
	87
	88	static bool __alloc_iova_check_used(struct interval_tree_span_iter *span,
	89	unsigned long length,
	90	unsigned long iova_alignment,
	91	unsigned long page_offset)
	92	{
	93	if (span->is_hole \|\| span->last_used - span->start_used < length - 1)
	94	return false;
	95
	96	span->start_used = ALIGN(span->start_used, iova_alignment) \|
	97	page_offset;
	98	if (span->start_used > span->last_used \|\|
	99	span->last_used - span->start_used < length - 1)
	100	return false;
	101	return true;
	102	}
	103
	104	/*
	105	* Automatically find a block of IOVA that is not being used and not reserved.
	106	* Does not return a 0 IOVA even if it is valid.
	107	*/
	108	static int iopt_alloc_iova(struct io_pagetable iopt, unsigned long iova,
	109	unsigned long uptr, unsigned long length)
	110	{
	111	unsigned long page_offset = uptr % PAGE_SIZE;
	112	struct interval_tree_double_span_iter used_span;
	113	struct interval_tree_span_iter allowed_span;
	114	unsigned long iova_alignment;
	115
	116	lockdep_assert_held(&iopt->iova_rwsem);
	117
	118	/* Protect roundup_pow-of_two() from overflow */
	119	if (length == 0 \|\| length >= ULONG_MAX / 2)
	120	return -EOVERFLOW;
	121
	122	/*
	123	* Keep alignment present in the uptr when building the IOVA, this
	124	* increases the chance we can map a THP.
	125	*/
	126	if (!uptr)
	127	iova_alignment = roundup_pow_of_two(length);
	128	else
	129	iova_alignment = min_t(unsigned long,
	130	roundup_pow_of_two(length),
	131	1UL << __ffs64(uptr));
	132
	133	if (iova_alignment < iopt->iova_alignment)
	134	return -EINVAL;
	135
	136	interval_tree_for_each_span(&allowed_span, &iopt->allowed_itree,
	137	PAGE_SIZE, ULONG_MAX - PAGE_SIZE) {
	138	if (RB_EMPTY_ROOT(&iopt->allowed_itree.rb_root)) {
	139	allowed_span.start_used = PAGE_SIZE;
	140	allowed_span.last_used = ULONG_MAX - PAGE_SIZE;
	141	allowed_span.is_hole = false;
	142	}
	143
	144	if (!__alloc_iova_check_used(&allowed_span, length,
	145	iova_alignment, page_offset))
	146	continue;
	147
	148	interval_tree_for_each_double_span(
	149	&used_span, &iopt->reserved_itree, &iopt->area_itree,
	150	allowed_span.start_used, allowed_span.last_used) {
	151	if (!__alloc_iova_check_hole(&used_span, length,
	152	iova_alignment,
	153	page_offset))
	154	continue;
	155
	156	*iova = used_span.start_hole;
	157	return 0;
	158	}
	159	}
	160	return -ENOSPC;
	161	}
	162
	163	static int iopt_check_iova(struct io_pagetable *iopt, unsigned long iova,
	164	unsigned long length)
	165	{
	166	unsigned long last;
	167
	168	lockdep_assert_held(&iopt->iova_rwsem);
	169
	170	if ((iova & (iopt->iova_alignment - 1)))
	171	return -EINVAL;
	172
	173	if (check_add_overflow(iova, length - 1, &last))
	174	return -EOVERFLOW;
	175
	176	/* No reserved IOVA intersects the range */
	177	if (iopt_reserved_iter_first(iopt, iova, last))
	178	return -EINVAL;
	179
	180	/* Check that there is not already a mapping in the range */
	181	if (iopt_area_iter_first(iopt, iova, last))
	182	return -EEXIST;
	183	return 0;
	184	}
	185
	186	/*
	187	* The area takes a slice of the pages from start_bytes to start_byte + length
	188	*/
	189	static int iopt_insert_area(struct io_pagetable iopt, struct iopt_area area,
	190	struct iopt_pages *pages, unsigned long iova,
	191	unsigned long start_byte, unsigned long length,
	192	int iommu_prot)
	193	{
	194	lockdep_assert_held_write(&iopt->iova_rwsem);
	195
	196	if ((iommu_prot & IOMMU_WRITE) && !pages->writable)
	197	return -EPERM;
	198
	199	area->iommu_prot = iommu_prot;
	200	area->page_offset = start_byte % PAGE_SIZE;
	201	if (area->page_offset & (iopt->iova_alignment - 1))
	202	return -EINVAL;
	203
	204	area->node.start = iova;
	205	if (check_add_overflow(iova, length - 1, &area->node.last))
	206	return -EOVERFLOW;
	207
	208	area->pages_node.start = start_byte / PAGE_SIZE;
	209	if (check_add_overflow(start_byte, length - 1, &area->pages_node.last))
	210	return -EOVERFLOW;
	211	area->pages_node.last = area->pages_node.last / PAGE_SIZE;
	212	if (WARN_ON(area->pages_node.last >= pages->npages))
	213	return -EOVERFLOW;
	214
	215	/*
	216	* The area is inserted with a NULL pages indicating it is not fully
	217	* initialized yet.
	218	*/
	219	area->iopt = iopt;
	220	interval_tree_insert(&area->node, &iopt->area_itree);
	221	return 0;
	222	}
	223
	224	static int iopt_alloc_area_pages(struct io_pagetable *iopt,
	225	struct list_head *pages_list,
	226	unsigned long length, unsigned long *dst_iova,
	227	int iommu_prot, unsigned int flags)
	228	{
	229	struct iopt_pages_list *elm;
	230	unsigned long iova;
	231	int rc = 0;
	232
	233	list_for_each_entry(elm, pages_list, next) {
	234	elm->area = kzalloc(sizeof(*elm->area), GFP_KERNEL_ACCOUNT);
	235	if (!elm->area)
	236	return -ENOMEM;
	237	}
	238
	239	down_write(&iopt->iova_rwsem);
	240	if ((length & (iopt->iova_alignment - 1)) \|\| !length) {
	241	rc = -EINVAL;
	242	goto out_unlock;
	243	}
	244
	245	if (flags & IOPT_ALLOC_IOVA) {
	246	/* Use the first entry to guess the ideal IOVA alignment */
	247	elm = list_first_entry(pages_list, struct iopt_pages_list,
	248	next);
	249	rc = iopt_alloc_iova(
	250	iopt, dst_iova,
	251	(uintptr_t)elm->pages->uptr + elm->start_byte, length);
	252	if (rc)
	253	goto out_unlock;
	254	if (IS_ENABLED(CONFIG_IOMMUFD_TEST) &&
	255	WARN_ON(iopt_check_iova(iopt, *dst_iova, length))) {
	256	rc = -EINVAL;
	257	goto out_unlock;
	258	}
	259	} else {
	260	rc = iopt_check_iova(iopt, *dst_iova, length);
	261	if (rc)
	262	goto out_unlock;
	263	}
	264
	265	/*
	266	* Areas are created with a NULL pages so that the IOVA space is
	267	* reserved and we can unlock the iova_rwsem.
	268	*/
	269	iova = *dst_iova;
	270	list_for_each_entry(elm, pages_list, next) {
	271	rc = iopt_insert_area(iopt, elm->area, elm->pages, iova,
	272	elm->start_byte, elm->length, iommu_prot);
	273	if (rc)
	274	goto out_unlock;
	275	iova += elm->length;
	276	}
	277
	278	out_unlock:
	279	up_write(&iopt->iova_rwsem);
	280	return rc;
	281	}
	282
	283	static void iopt_abort_area(struct iopt_area *area)
	284	{
	285	if (IS_ENABLED(CONFIG_IOMMUFD_TEST))
	286	WARN_ON(area->pages);
	287	if (area->iopt) {
	288	down_write(&area->iopt->iova_rwsem);
	289	interval_tree_remove(&area->node, &area->iopt->area_itree);
	290	up_write(&area->iopt->iova_rwsem);
	291	}
	292	kfree(area);
	293	}
	294
	295	void iopt_free_pages_list(struct list_head *pages_list)
	296	{
	297	struct iopt_pages_list *elm;
	298
	299	while ((elm = list_first_entry_or_null(pages_list,
	300	struct iopt_pages_list, next))) {
	301	if (elm->area)
	302	iopt_abort_area(elm->area);
	303	if (elm->pages)
	304	iopt_put_pages(elm->pages);
	305	list_del(&elm->next);
	306	kfree(elm);
	307	}
	308	}
	309
	310	static int iopt_fill_domains_pages(struct list_head *pages_list)
	311	{
	312	struct iopt_pages_list *undo_elm;
	313	struct iopt_pages_list *elm;
	314	int rc;
	315
	316	list_for_each_entry(elm, pages_list, next) {
	317	rc = iopt_area_fill_domains(elm->area, elm->pages);
	318	if (rc)
	319	goto err_undo;
	320	}
	321	return 0;
	322
	323	err_undo:
	324	list_for_each_entry(undo_elm, pages_list, next) {
	325	if (undo_elm == elm)
	326	break;
	327	iopt_area_unfill_domains(undo_elm->area, undo_elm->pages);
	328	}
	329	return rc;
	330	}
	331
	332	int iopt_map_pages(struct io_pagetable iopt, struct list_head pages_list,
	333	unsigned long length, unsigned long *dst_iova,
	334	int iommu_prot, unsigned int flags)
	335	{
	336	struct iopt_pages_list *elm;
	337	int rc;
	338
	339	rc = iopt_alloc_area_pages(iopt, pages_list, length, dst_iova,
	340	iommu_prot, flags);
	341	if (rc)
	342	return rc;
	343
	344	down_read(&iopt->domains_rwsem);
	345	rc = iopt_fill_domains_pages(pages_list);
	346	if (rc)
	347	goto out_unlock_domains;
	348
	349	down_write(&iopt->iova_rwsem);
	350	list_for_each_entry(elm, pages_list, next) {
	351	/*
	352	* area->pages must be set inside the domains_rwsem to ensure
	353	* any newly added domains will get filled. Moves the reference
	354	* in from the list.
	355	*/
	356	elm->area->pages = elm->pages;
	357	elm->pages = NULL;
	358	elm->area = NULL;
	359	}
	360	up_write(&iopt->iova_rwsem);
	361	out_unlock_domains:
	362	up_read(&iopt->domains_rwsem);
	363	return rc;
	364	}
	365
	366	/**
	367	* iopt_map_user_pages() - Map a user VA to an iova in the io page table
	368	* @ictx: iommufd_ctx the iopt is part of
	369	* @iopt: io_pagetable to act on
	370	* @iova: If IOPT_ALLOC_IOVA is set this is unused on input and contains
	371	* the chosen iova on output. Otherwise is the iova to map to on input
	372	* @uptr: User VA to map
	373	* @length: Number of bytes to map
	374	* @iommu_prot: Combination of IOMMU_READ/WRITE/etc bits for the mapping
	375	* @flags: IOPT_ALLOC_IOVA or zero
	376	*
	377	* iova, uptr, and length must be aligned to iova_alignment. For domain backed
	378	* page tables this will pin the pages and load them into the domain at iova.
	379	* For non-domain page tables this will only setup a lazy reference and the
	380	* caller must use iopt_access_pages() to touch them.
	381	*
	382	* iopt_unmap_iova() must be called to undo this before the io_pagetable can be
	383	* destroyed.
	384	*/
	385	int iopt_map_user_pages(struct iommufd_ctx ictx, struct io_pagetable iopt,
	386	unsigned long iova, void __user uptr,
	387	unsigned long length, int iommu_prot,
	388	unsigned int flags)
	389	{
	390	struct iopt_pages_list elm = {};
	391	LIST_HEAD(pages_list);
	392	int rc;
	393
	394	elm.pages = iopt_alloc_pages(uptr, length, iommu_prot & IOMMU_WRITE);
	395	if (IS_ERR(elm.pages))
	396	return PTR_ERR(elm.pages);
	397	if (ictx->account_mode == IOPT_PAGES_ACCOUNT_MM &&
	398	elm.pages->account_mode == IOPT_PAGES_ACCOUNT_USER)
	399	elm.pages->account_mode = IOPT_PAGES_ACCOUNT_MM;
	400	elm.start_byte = uptr - elm.pages->uptr;
	401	elm.length = length;
	402	list_add(&elm.next, &pages_list);
	403
	404	rc = iopt_map_pages(iopt, &pages_list, length, iova, iommu_prot, flags);
	405	if (rc) {
	406	if (elm.area)
	407	iopt_abort_area(elm.area);
	408	if (elm.pages)
	409	iopt_put_pages(elm.pages);
	410	return rc;
	411	}
	412	return 0;
	413	}
	414
	415	int iopt_get_pages(struct io_pagetable *iopt, unsigned long iova,
	416	unsigned long length, struct list_head *pages_list)
	417	{
	418	struct iopt_area_contig_iter iter;
	419	unsigned long last_iova;
	420	struct iopt_area *area;
	421	int rc;
	422
	423	if (!length)
	424	return -EINVAL;
	425	if (check_add_overflow(iova, length - 1, &last_iova))
	426	return -EOVERFLOW;
	427
	428	down_read(&iopt->iova_rwsem);
	429	iopt_for_each_contig_area(&iter, area, iopt, iova, last_iova) {
	430	struct iopt_pages_list *elm;
	431	unsigned long last = min(last_iova, iopt_area_last_iova(area));
	432
	433	elm = kzalloc(sizeof(*elm), GFP_KERNEL_ACCOUNT);
	434	if (!elm) {
	435	rc = -ENOMEM;
	436	goto err_free;
	437	}
	438	elm->start_byte = iopt_area_start_byte(area, iter.cur_iova);
	439	elm->pages = area->pages;
	440	elm->length = (last - iter.cur_iova) + 1;
	441	kref_get(&elm->pages->kref);
	442	list_add_tail(&elm->next, pages_list);
	443	}
	444	if (!iopt_area_contig_done(&iter)) {
	445	rc = -ENOENT;
	446	goto err_free;
	447	}
	448	up_read(&iopt->iova_rwsem);
	449	return 0;
	450	err_free:
	451	up_read(&iopt->iova_rwsem);
	452	iopt_free_pages_list(pages_list);
	453	return rc;
	454	}
	455
	456	static int iopt_unmap_iova_range(struct io_pagetable *iopt, unsigned long start,
	457	unsigned long last, unsigned long *unmapped)
	458	{
	459	struct iopt_area *area;
	460	unsigned long unmapped_bytes = 0;
	461	int rc = -ENOENT;
	462
	463	/*
	464	* The domains_rwsem must be held in read mode any time any area->pages
	465	* is NULL. This prevents domain attach/detatch from running
	466	* concurrently with cleaning up the area.
	467	*/
	468	again:
	469	down_read(&iopt->domains_rwsem);
	470	down_write(&iopt->iova_rwsem);
	471	while ((area = iopt_area_iter_first(iopt, start, last))) {
	472	unsigned long area_last = iopt_area_last_iova(area);
	473	unsigned long area_first = iopt_area_iova(area);
	474	struct iopt_pages *pages;
	475
	476	/* Userspace should not race map/unmap's of the same area */
	477	if (!area->pages) {
	478	rc = -EBUSY;
	479	goto out_unlock_iova;
	480	}
	481
	482	if (area_first < start \|\| area_last > last) {
	483	rc = -ENOENT;
	484	goto out_unlock_iova;
	485	}
	486
	487	/*
	488	* num_accesses writers must hold the iova_rwsem too, so we can
	489	* safely read it under the write side of the iovam_rwsem
	490	* without the pages->mutex.
	491	*/
	492	if (area->num_accesses) {
	493	start = area_first;
	494	area->prevent_access = true;
	495	up_write(&iopt->iova_rwsem);
	496	up_read(&iopt->domains_rwsem);
	497	iommufd_access_notify_unmap(iopt, area_first,
	498	iopt_area_length(area));
	499	if (WARN_ON(READ_ONCE(area->num_accesses)))
	500	return -EDEADLOCK;
	501	goto again;
	502	}
	503
	504	pages = area->pages;
	505	area->pages = NULL;
	506	up_write(&iopt->iova_rwsem);
	507
	508	iopt_area_unfill_domains(area, pages);
	509	iopt_abort_area(area);
	510	iopt_put_pages(pages);
	511
	512	unmapped_bytes += area_last - area_first + 1;
	513
	514	down_write(&iopt->iova_rwsem);
	515	}
	516	if (unmapped_bytes)
	517	rc = 0;
	518
	519	out_unlock_iova:
	520	up_write(&iopt->iova_rwsem);
	521	up_read(&iopt->domains_rwsem);
	522	if (unmapped)
	523	*unmapped = unmapped_bytes;
	524	return rc;
	525	}
	526
	527	/**
	528	* iopt_unmap_iova() - Remove a range of iova
	529	* @iopt: io_pagetable to act on
	530	* @iova: Starting iova to unmap
	531	* @length: Number of bytes to unmap
	532	* @unmapped: Return number of bytes unmapped
	533	*
	534	* The requested range must be a superset of existing ranges.
	535	* Splitting/truncating IOVA mappings is not allowed.
	536	*/
	537	int iopt_unmap_iova(struct io_pagetable *iopt, unsigned long iova,
	538	unsigned long length, unsigned long *unmapped)
	539	{
	540	unsigned long iova_last;
	541
	542	if (!length)
	543	return -EINVAL;
	544
	545	if (check_add_overflow(iova, length - 1, &iova_last))
	546	return -EOVERFLOW;
	547
	548	return iopt_unmap_iova_range(iopt, iova, iova_last, unmapped);
	549	}
	550
	551	int iopt_unmap_all(struct io_pagetable iopt, unsigned long unmapped)
	552	{
	553	int rc;
	554
	555	rc = iopt_unmap_iova_range(iopt, 0, ULONG_MAX, unmapped);
	556	/* If the IOVAs are empty then unmap all succeeds */
	557	if (rc == -ENOENT)
	558	return 0;
	559	return rc;
	560	}
	561
	562	/* The caller must always free all the nodes in the allowed_iova rb_root. */
	563	int iopt_set_allow_iova(struct io_pagetable *iopt,
	564	struct rb_root_cached *allowed_iova)
	565	{
	566	struct iopt_allowed *allowed;
	567
	568	down_write(&iopt->iova_rwsem);
	569	swap(*allowed_iova, iopt->allowed_itree);
	570
	571	for (allowed = iopt_allowed_iter_first(iopt, 0, ULONG_MAX); allowed;
	572	allowed = iopt_allowed_iter_next(allowed, 0, ULONG_MAX)) {
	573	if (iopt_reserved_iter_first(iopt, allowed->node.start,
	574	allowed->node.last)) {
	575	swap(*allowed_iova, iopt->allowed_itree);
	576	up_write(&iopt->iova_rwsem);
	577	return -EADDRINUSE;
	578	}
	579	}
	580	up_write(&iopt->iova_rwsem);
	581	return 0;
	582	}
	583
	584	int iopt_reserve_iova(struct io_pagetable *iopt, unsigned long start,
	585	unsigned long last, void *owner)
	586	{
	587	struct iopt_reserved *reserved;
	588
	589	lockdep_assert_held_write(&iopt->iova_rwsem);
	590
	591	if (iopt_area_iter_first(iopt, start, last) \|\|
	592	iopt_allowed_iter_first(iopt, start, last))
	593	return -EADDRINUSE;
	594
	595	reserved = kzalloc(sizeof(*reserved), GFP_KERNEL_ACCOUNT);
	596	if (!reserved)
	597	return -ENOMEM;
	598	reserved->node.start = start;
	599	reserved->node.last = last;
	600	reserved->owner = owner;
	601	interval_tree_insert(&reserved->node, &iopt->reserved_itree);
	602	return 0;
	603	}
	604
	605	static void __iopt_remove_reserved_iova(struct io_pagetable iopt, void owner)
	606	{
	607	struct iopt_reserved reserved, next;
	608
	609	lockdep_assert_held_write(&iopt->iova_rwsem);
	610
	611	for (reserved = iopt_reserved_iter_first(iopt, 0, ULONG_MAX); reserved;
	612	reserved = next) {
	613	next = iopt_reserved_iter_next(reserved, 0, ULONG_MAX);
	614
	615	if (reserved->owner == owner) {
	616	interval_tree_remove(&reserved->node,
	617	&iopt->reserved_itree);
	618	kfree(reserved);
	619	}
	620	}
	621	}
	622
	623	void iopt_remove_reserved_iova(struct io_pagetable iopt, void owner)
	624	{
	625	down_write(&iopt->iova_rwsem);
	626	__iopt_remove_reserved_iova(iopt, owner);
	627	up_write(&iopt->iova_rwsem);
	628	}
	629
	630	void iopt_init_table(struct io_pagetable *iopt)
	631	{
	632	init_rwsem(&iopt->iova_rwsem);
	633	init_rwsem(&iopt->domains_rwsem);
	634	iopt->area_itree = RB_ROOT_CACHED;
	635	iopt->allowed_itree = RB_ROOT_CACHED;
	636	iopt->reserved_itree = RB_ROOT_CACHED;
	637	xa_init_flags(&iopt->domains, XA_FLAGS_ACCOUNT);
	638	xa_init_flags(&iopt->access_list, XA_FLAGS_ALLOC);
	639
	640	/*
	641	* iopt's start as SW tables that can use the entire size_t IOVA space
	642	* due to the use of size_t in the APIs. They have no alignment
	643	* restriction.
	644	*/
	645	iopt->iova_alignment = 1;
	646	}
	647
	648	void iopt_destroy_table(struct io_pagetable *iopt)
	649	{
	650	struct interval_tree_node *node;
	651
	652	if (IS_ENABLED(CONFIG_IOMMUFD_TEST))
	653	iopt_remove_reserved_iova(iopt, NULL);
	654
	655	while ((node = interval_tree_iter_first(&iopt->allowed_itree, 0,
	656	ULONG_MAX))) {
	657	interval_tree_remove(node, &iopt->allowed_itree);
	658	kfree(container_of(node, struct iopt_allowed, node));
	659	}
	660
	661	WARN_ON(!RB_EMPTY_ROOT(&iopt->reserved_itree.rb_root));
	662	WARN_ON(!xa_empty(&iopt->domains));
	663	WARN_ON(!xa_empty(&iopt->access_list));
	664	WARN_ON(!RB_EMPTY_ROOT(&iopt->area_itree.rb_root));
	665	}
	666
	667	/**
	668	* iopt_unfill_domain() - Unfill a domain with PFNs
	669	* @iopt: io_pagetable to act on
	670	* @domain: domain to unfill
	671	*
	672	* This is used when removing a domain from the iopt. Every area in the iopt
	673	* will be unmapped from the domain. The domain must already be removed from the
	674	* domains xarray.
	675	*/
	676	static void iopt_unfill_domain(struct io_pagetable *iopt,
	677	struct iommu_domain *domain)
	678	{
	679	struct iopt_area *area;
	680
	681	lockdep_assert_held(&iopt->iova_rwsem);
	682	lockdep_assert_held_write(&iopt->domains_rwsem);
	683
	684	/*
	685	* Some other domain is holding all the pfns still, rapidly unmap this
	686	* domain.
	687	*/
	688	if (iopt->next_domain_id != 0) {
	689	/* Pick an arbitrary remaining domain to act as storage */
	690	struct iommu_domain *storage_domain =
	691	xa_load(&iopt->domains, 0);
	692
	693	for (area = iopt_area_iter_first(iopt, 0, ULONG_MAX); area;
	694	area = iopt_area_iter_next(area, 0, ULONG_MAX)) {
	695	struct iopt_pages *pages = area->pages;
	696
	697	if (!pages)
	698	continue;
	699
	700	mutex_lock(&pages->mutex);
	701	if (IS_ENABLED(CONFIG_IOMMUFD_TEST))
	702	WARN_ON(!area->storage_domain);
	703	if (area->storage_domain == domain)
	704	area->storage_domain = storage_domain;
	705	mutex_unlock(&pages->mutex);
	706
	707	iopt_area_unmap_domain(area, domain);
	708	}
	709	return;
	710	}
	711
	712	for (area = iopt_area_iter_first(iopt, 0, ULONG_MAX); area;
	713	area = iopt_area_iter_next(area, 0, ULONG_MAX)) {
	714	struct iopt_pages *pages = area->pages;
	715
	716	if (!pages)
	717	continue;
	718
	719	mutex_lock(&pages->mutex);
	720	interval_tree_remove(&area->pages_node, &pages->domains_itree);
	721	WARN_ON(area->storage_domain != domain);
	722	area->storage_domain = NULL;
	723	iopt_area_unfill_domain(area, pages, domain);
	724	mutex_unlock(&pages->mutex);
	725	}
	726	}
	727
	728	/**
	729	* iopt_fill_domain() - Fill a domain with PFNs
	730	* @iopt: io_pagetable to act on
	731	* @domain: domain to fill
	732	*
	733	* Fill the domain with PFNs from every area in the iopt. On failure the domain
	734	* is left unchanged.
	735	*/
	736	static int iopt_fill_domain(struct io_pagetable *iopt,
	737	struct iommu_domain *domain)
	738	{
	739	struct iopt_area *end_area;
	740	struct iopt_area *area;
	741	int rc;
	742
	743	lockdep_assert_held(&iopt->iova_rwsem);
	744	lockdep_assert_held_write(&iopt->domains_rwsem);
	745
	746	for (area = iopt_area_iter_first(iopt, 0, ULONG_MAX); area;
	747	area = iopt_area_iter_next(area, 0, ULONG_MAX)) {
	748	struct iopt_pages *pages = area->pages;
	749
	750	if (!pages)
	751	continue;
	752
	753	mutex_lock(&pages->mutex);
	754	rc = iopt_area_fill_domain(area, domain);
	755	if (rc) {
	756	mutex_unlock(&pages->mutex);
	757	goto out_unfill;
	758	}
	759	if (!area->storage_domain) {
	760	WARN_ON(iopt->next_domain_id != 0);
	761	area->storage_domain = domain;
	762	interval_tree_insert(&area->pages_node,
	763	&pages->domains_itree);
	764	}
	765	mutex_unlock(&pages->mutex);
	766	}
	767	return 0;
	768
	769	out_unfill:
	770	end_area = area;
	771	for (area = iopt_area_iter_first(iopt, 0, ULONG_MAX); area;
	772	area = iopt_area_iter_next(area, 0, ULONG_MAX)) {
	773	struct iopt_pages *pages = area->pages;
	774
	775	if (area == end_area)
	776	break;
	777	if (!pages)
	778	continue;
	779	mutex_lock(&pages->mutex);
	780	if (iopt->next_domain_id == 0) {
	781	interval_tree_remove(&area->pages_node,
	782	&pages->domains_itree);
	783	area->storage_domain = NULL;
	784	}
	785	iopt_area_unfill_domain(area, pages, domain);
	786	mutex_unlock(&pages->mutex);
	787	}
	788	return rc;
	789	}
	790
	791	/* All existing area's conform to an increased page size */
	792	static int iopt_check_iova_alignment(struct io_pagetable *iopt,
	793	unsigned long new_iova_alignment)
	794	{
	795	unsigned long align_mask = new_iova_alignment - 1;
	796	struct iopt_area *area;
	797
	798	lockdep_assert_held(&iopt->iova_rwsem);
	799	lockdep_assert_held(&iopt->domains_rwsem);
	800
	801	for (area = iopt_area_iter_first(iopt, 0, ULONG_MAX); area;
	802	area = iopt_area_iter_next(area, 0, ULONG_MAX))
	803	if ((iopt_area_iova(area) & align_mask) \|\|
	804	(iopt_area_length(area) & align_mask) \|\|
	805	(area->page_offset & align_mask))
	806	return -EADDRINUSE;
	807
	808	if (IS_ENABLED(CONFIG_IOMMUFD_TEST)) {
	809	struct iommufd_access *access;
	810	unsigned long index;
	811
	812	xa_for_each(&iopt->access_list, index, access)
	813	if (WARN_ON(access->iova_alignment >
	814	new_iova_alignment))
	815	return -EADDRINUSE;
	816	}
	817	return 0;
	818	}
	819
	820	int iopt_table_add_domain(struct io_pagetable *iopt,
	821	struct iommu_domain *domain)
	822	{
	823	const struct iommu_domain_geometry *geometry = &domain->geometry;
	824	struct iommu_domain *iter_domain;
	825	unsigned int new_iova_alignment;
	826	unsigned long index;
	827	int rc;
	828
	829	down_write(&iopt->domains_rwsem);
	830	down_write(&iopt->iova_rwsem);
	831
	832	xa_for_each(&iopt->domains, index, iter_domain) {
	833	if (WARN_ON(iter_domain == domain)) {
	834	rc = -EEXIST;
	835	goto out_unlock;
	836	}
	837	}
	838
	839	/*
	840	* The io page size drives the iova_alignment. Internally the iopt_pages
	841	* works in PAGE_SIZE units and we adjust when mapping sub-PAGE_SIZE
	842	* objects into the iommu_domain.
	843	*
	844	* A iommu_domain must always be able to accept PAGE_SIZE to be
	845	* compatible as we can't guarantee higher contiguity.
	846	*/
	847	new_iova_alignment = max_t(unsigned long,
	848	1UL << __ffs(domain->pgsize_bitmap),
	849	iopt->iova_alignment);
	850	if (new_iova_alignment > PAGE_SIZE) {
	851	rc = -EINVAL;
	852	goto out_unlock;
	853	}
	854	if (new_iova_alignment != iopt->iova_alignment) {
	855	rc = iopt_check_iova_alignment(iopt, new_iova_alignment);
	856	if (rc)
	857	goto out_unlock;
	858	}
	859
	860	/* No area exists that is outside the allowed domain aperture */
	861	if (geometry->aperture_start != 0) {
	862	rc = iopt_reserve_iova(iopt, 0, geometry->aperture_start - 1,
	863	domain);
	864	if (rc)
	865	goto out_reserved;
	866	}
	867	if (geometry->aperture_end != ULONG_MAX) {
	868	rc = iopt_reserve_iova(iopt, geometry->aperture_end + 1,
	869	ULONG_MAX, domain);
	870	if (rc)
	871	goto out_reserved;
	872	}
	873
	874	rc = xa_reserve(&iopt->domains, iopt->next_domain_id, GFP_KERNEL);
	875	if (rc)
	876	goto out_reserved;
	877
	878	rc = iopt_fill_domain(iopt, domain);
	879	if (rc)
	880	goto out_release;
	881
	882	iopt->iova_alignment = new_iova_alignment;
	883	xa_store(&iopt->domains, iopt->next_domain_id, domain, GFP_KERNEL);
	884	iopt->next_domain_id++;
	885	up_write(&iopt->iova_rwsem);
	886	up_write(&iopt->domains_rwsem);
	887	return 0;
	888	out_release:
	889	xa_release(&iopt->domains, iopt->next_domain_id);
	890	out_reserved:
	891	__iopt_remove_reserved_iova(iopt, domain);
	892	out_unlock:
	893	up_write(&iopt->iova_rwsem);
	894	up_write(&iopt->domains_rwsem);
	895	return rc;
	896	}
	897
	898	static int iopt_calculate_iova_alignment(struct io_pagetable *iopt)
	899	{
	900	unsigned long new_iova_alignment;
	901	struct iommufd_access *access;
	902	struct iommu_domain *domain;
	903	unsigned long index;
	904
	905	lockdep_assert_held_write(&iopt->iova_rwsem);
	906	lockdep_assert_held(&iopt->domains_rwsem);
	907
	908	/* See batch_iommu_map_small() */
	909	if (iopt->disable_large_pages)
	910	new_iova_alignment = PAGE_SIZE;
	911	else
	912	new_iova_alignment = 1;
	913
	914	xa_for_each(&iopt->domains, index, domain)
	915	new_iova_alignment = max_t(unsigned long,
	916	1UL << __ffs(domain->pgsize_bitmap),
	917	new_iova_alignment);
	918	xa_for_each(&iopt->access_list, index, access)
	919	new_iova_alignment = max_t(unsigned long,
	920	access->iova_alignment,
	921	new_iova_alignment);
	922
	923	if (new_iova_alignment > iopt->iova_alignment) {
	924	int rc;
	925
	926	rc = iopt_check_iova_alignment(iopt, new_iova_alignment);
	927	if (rc)
	928	return rc;
	929	}
	930	iopt->iova_alignment = new_iova_alignment;
	931	return 0;
	932	}
	933
	934	void iopt_table_remove_domain(struct io_pagetable *iopt,
	935	struct iommu_domain *domain)
	936	{
	937	struct iommu_domain *iter_domain = NULL;
	938	unsigned long index;
	939
	940	down_write(&iopt->domains_rwsem);
	941	down_write(&iopt->iova_rwsem);
	942
	943	xa_for_each(&iopt->domains, index, iter_domain)
	944	if (iter_domain == domain)
	945	break;
	946	if (WARN_ON(iter_domain != domain) \|\| index >= iopt->next_domain_id)
	947	goto out_unlock;
	948
	949	/*
	950	* Compress the xarray to keep it linear by swapping the entry to erase
	951	* with the tail entry and shrinking the tail.
	952	*/
	953	iopt->next_domain_id--;
	954	iter_domain = xa_erase(&iopt->domains, iopt->next_domain_id);
	955	if (index != iopt->next_domain_id)
	956	xa_store(&iopt->domains, index, iter_domain, GFP_KERNEL);
	957
	958	iopt_unfill_domain(iopt, domain);
	959	__iopt_remove_reserved_iova(iopt, domain);
	960
	961	WARN_ON(iopt_calculate_iova_alignment(iopt));
	962	out_unlock:
	963	up_write(&iopt->iova_rwsem);
	964	up_write(&iopt->domains_rwsem);
	965	}
	966
	967	/**
	968	* iopt_area_split - Split an area into two parts at iova
	969	* @area: The area to split
	970	* @iova: Becomes the last of a new area
	971	*
	972	* This splits an area into two. It is part of the VFIO compatibility to allow
	973	* poking a hole in the mapping. The two areas continue to point at the same
	974	* iopt_pages, just with different starting bytes.
	975	*/
	976	static int iopt_area_split(struct iopt_area *area, unsigned long iova)
	977	{
	978	unsigned long alignment = area->iopt->iova_alignment;
	979	unsigned long last_iova = iopt_area_last_iova(area);
	980	unsigned long start_iova = iopt_area_iova(area);
	981	unsigned long new_start = iova + 1;
	982	struct io_pagetable *iopt = area->iopt;
	983	struct iopt_pages *pages = area->pages;
	984	struct iopt_area *lhs;
	985	struct iopt_area *rhs;
	986	int rc;
	987
	988	lockdep_assert_held_write(&iopt->iova_rwsem);
	989
	990	if (iova == start_iova \|\| iova == last_iova)
	991	return 0;
	992
	993	if (!pages \|\| area->prevent_access)
	994	return -EBUSY;
	995
	996	if (new_start & (alignment - 1) \|\|
	997	iopt_area_start_byte(area, new_start) & (alignment - 1))
	998	return -EINVAL;
	999
	1000	lhs = kzalloc(sizeof(*area), GFP_KERNEL_ACCOUNT);
	1001	if (!lhs)
	1002	return -ENOMEM;
	1003
	1004	rhs = kzalloc(sizeof(*area), GFP_KERNEL_ACCOUNT);
	1005	if (!rhs) {
	1006	rc = -ENOMEM;
	1007	goto err_free_lhs;
	1008	}
	1009
	1010	mutex_lock(&pages->mutex);
	1011	/*
	1012	* Splitting is not permitted if an access exists, we don't track enough
	1013	* information to split existing accesses.
	1014	*/
	1015	if (area->num_accesses) {
	1016	rc = -EINVAL;
	1017	goto err_unlock;
	1018	}
	1019
	1020	/*
	1021	* Splitting is not permitted if a domain could have been mapped with
	1022	* huge pages.
	1023	*/
	1024	if (area->storage_domain && !iopt->disable_large_pages) {
	1025	rc = -EINVAL;
	1026	goto err_unlock;
	1027	}
	1028
	1029	interval_tree_remove(&area->node, &iopt->area_itree);
	1030	rc = iopt_insert_area(iopt, lhs, area->pages, start_iova,
	1031	iopt_area_start_byte(area, start_iova),
	1032	(new_start - 1) - start_iova + 1,
	1033	area->iommu_prot);
	1034	if (WARN_ON(rc))
	1035	goto err_insert;
	1036
	1037	rc = iopt_insert_area(iopt, rhs, area->pages, new_start,
	1038	iopt_area_start_byte(area, new_start),
	1039	last_iova - new_start + 1, area->iommu_prot);
	1040	if (WARN_ON(rc))
	1041	goto err_remove_lhs;
	1042
	1043	lhs->storage_domain = area->storage_domain;
	1044	lhs->pages = area->pages;
	1045	rhs->storage_domain = area->storage_domain;
	1046	rhs->pages = area->pages;
	1047	kref_get(&rhs->pages->kref);
	1048	kfree(area);
	1049	mutex_unlock(&pages->mutex);
	1050
	1051	/*
	1052	* No change to domains or accesses because the pages hasn't been
	1053	* changed
	1054	*/
	1055	return 0;
	1056
	1057	err_remove_lhs:
	1058	interval_tree_remove(&lhs->node, &iopt->area_itree);
	1059	err_insert:
	1060	interval_tree_insert(&area->node, &iopt->area_itree);
	1061	err_unlock:
	1062	mutex_unlock(&pages->mutex);
	1063	kfree(rhs);
	1064	err_free_lhs:
	1065	kfree(lhs);
	1066	return rc;
	1067	}
	1068
	1069	int iopt_cut_iova(struct io_pagetable iopt, unsigned long iovas,
	1070	size_t num_iovas)
	1071	{
	1072	int rc = 0;
	1073	int i;
	1074
	1075	down_write(&iopt->iova_rwsem);
	1076	for (i = 0; i < num_iovas; i++) {
	1077	struct iopt_area *area;
	1078
	1079	area = iopt_area_iter_first(iopt, iovas[i], iovas[i]);
	1080	if (!area)
	1081	continue;
	1082	rc = iopt_area_split(area, iovas[i]);
	1083	if (rc)
	1084	break;
	1085	}
	1086	up_write(&iopt->iova_rwsem);
	1087	return rc;
	1088	}
	1089
	1090	void iopt_enable_large_pages(struct io_pagetable *iopt)
	1091	{
	1092	int rc;
	1093
	1094	down_write(&iopt->domains_rwsem);
	1095	down_write(&iopt->iova_rwsem);
	1096	WRITE_ONCE(iopt->disable_large_pages, false);
	1097	rc = iopt_calculate_iova_alignment(iopt);
	1098	WARN_ON(rc);
	1099	up_write(&iopt->iova_rwsem);
	1100	up_write(&iopt->domains_rwsem);
	1101	}
	1102
	1103	int iopt_disable_large_pages(struct io_pagetable *iopt)
	1104	{
	1105	int rc = 0;
	1106
	1107	down_write(&iopt->domains_rwsem);
	1108	down_write(&iopt->iova_rwsem);
	1109	if (iopt->disable_large_pages)
	1110	goto out_unlock;
	1111
	1112	/* Won't do it if domains already have pages mapped in them */
	1113	if (!xa_empty(&iopt->domains) &&
	1114	!RB_EMPTY_ROOT(&iopt->area_itree.rb_root)) {
	1115	rc = -EINVAL;
	1116	goto out_unlock;
	1117	}
	1118
	1119	WRITE_ONCE(iopt->disable_large_pages, true);
	1120	rc = iopt_calculate_iova_alignment(iopt);
	1121	if (rc)
	1122	WRITE_ONCE(iopt->disable_large_pages, false);
	1123	out_unlock:
	1124	up_write(&iopt->iova_rwsem);
	1125	up_write(&iopt->domains_rwsem);
	1126	return rc;
	1127	}
	1128
	1129	int iopt_add_access(struct io_pagetable iopt, struct iommufd_access access)
	1130	{
	1131	int rc;
	1132
	1133	down_write(&iopt->domains_rwsem);
	1134	down_write(&iopt->iova_rwsem);
	1135	rc = xa_alloc(&iopt->access_list, &access->iopt_access_list_id, access,
	1136	xa_limit_16b, GFP_KERNEL_ACCOUNT);
	1137	if (rc)
	1138	goto out_unlock;
	1139
	1140	rc = iopt_calculate_iova_alignment(iopt);
	1141	if (rc) {
	1142	xa_erase(&iopt->access_list, access->iopt_access_list_id);
	1143	goto out_unlock;
	1144	}
	1145
	1146	out_unlock:
	1147	up_write(&iopt->iova_rwsem);
	1148	up_write(&iopt->domains_rwsem);
	1149	return rc;
	1150	}
	1151
	1152	void iopt_remove_access(struct io_pagetable *iopt,
	1153	struct iommufd_access *access)
	1154	{
	1155	down_write(&iopt->domains_rwsem);
	1156	down_write(&iopt->iova_rwsem);
	1157	WARN_ON(xa_erase(&iopt->access_list, access->iopt_access_list_id) !=
	1158	access);
	1159	WARN_ON(iopt_calculate_iova_alignment(iopt));
	1160	up_write(&iopt->iova_rwsem);
	1161	up_write(&iopt->domains_rwsem);
	1162	}
	1163
	1164	/* Narrow the valid_iova_itree to include reserved ranges from a group. */
	1165	int iopt_table_enforce_group_resv_regions(struct io_pagetable *iopt,
	1166	struct device *device,
	1167	struct iommu_group *group,
	1168	phys_addr_t *sw_msi_start)
	1169	{
	1170	struct iommu_resv_region *resv;
	1171	struct iommu_resv_region *tmp;
	1172	LIST_HEAD(group_resv_regions);
	1173	unsigned int num_hw_msi = 0;
	1174	unsigned int num_sw_msi = 0;
	1175	int rc;
	1176
	1177	down_write(&iopt->iova_rwsem);
	1178	rc = iommu_get_group_resv_regions(group, &group_resv_regions);
	1179	if (rc)
	1180	goto out_unlock;
	1181
	1182	list_for_each_entry(resv, &group_resv_regions, list) {
	1183	if (resv->type == IOMMU_RESV_DIRECT_RELAXABLE)
	1184	continue;
	1185
	1186	if (sw_msi_start && resv->type == IOMMU_RESV_MSI)
	1187	num_hw_msi++;
	1188	if (sw_msi_start && resv->type == IOMMU_RESV_SW_MSI) {
	1189	*sw_msi_start = resv->start;
	1190	num_sw_msi++;
	1191	}
	1192
	1193	rc = iopt_reserve_iova(iopt, resv->start,
	1194	resv->length - 1 + resv->start, device);
	1195	if (rc)
	1196	goto out_reserved;
	1197	}
	1198
	1199	/* Drivers must offer sane combinations of regions */
	1200	if (WARN_ON(num_sw_msi && num_hw_msi) \|\| WARN_ON(num_sw_msi > 1)) {
	1201	rc = -EINVAL;
	1202	goto out_reserved;
	1203	}
	1204
	1205	rc = 0;
	1206	goto out_free_resv;
	1207
	1208	out_reserved:
	1209	__iopt_remove_reserved_iova(iopt, device);
	1210	out_free_resv:
	1211	list_for_each_entry_safe(resv, tmp, &group_resv_regions, list)
	1212	kfree(resv);
	1213	out_unlock:
	1214	up_write(&iopt->iova_rwsem);
	1215	return rc;
	1216	}

drivers/iommu/iommufd/io_pagetable.h+241−0

@@ -0,0 +1,241 @@
	1	/* SPDX-License-Identifier: GPL-2.0 */
	2	/* Copyright (c) 2021-2022, NVIDIA CORPORATION & AFFILIATES.
	3	*
	4	*/
	5	#ifndef __IO_PAGETABLE_H
	6	#define __IO_PAGETABLE_H
	7
	8	#include <linux/interval_tree.h>
	9	#include <linux/mutex.h>
	10	#include <linux/kref.h>
	11	#include <linux/xarray.h>
	12
	13	#include "iommufd_private.h"
	14
	15	struct iommu_domain;
	16
	17	/*
	18	* Each io_pagetable is composed of intervals of areas which cover regions of
	19	* the iova that are backed by something. iova not covered by areas is not
	20	* populated in the page table. Each area is fully populated with pages.
	21	*
	22	* iovas are in byte units, but must be iopt->iova_alignment aligned.
	23	*
	24	* pages can be NULL, this means some other thread is still working on setting
	25	* up or tearing down the area. When observed under the write side of the
	26	* domain_rwsem a NULL pages must mean the area is still being setup and no
	27	* domains are filled.
	28	*
	29	* storage_domain points at an arbitrary iommu_domain that is holding the PFNs
	30	* for this area. It is locked by the pages->mutex. This simplifies the locking
	31	* as the pages code can rely on the storage_domain without having to get the
	32	* iopt->domains_rwsem.
	33	*
	34	* The io_pagetable::iova_rwsem protects node
	35	* The iopt_pages::mutex protects pages_node
	36	* iopt and iommu_prot are immutable
	37	* The pages::mutex protects num_accesses
	38	*/
	39	struct iopt_area {
	40	struct interval_tree_node node;
	41	struct interval_tree_node pages_node;
	42	struct io_pagetable *iopt;
	43	struct iopt_pages *pages;
	44	struct iommu_domain *storage_domain;
	45	/* How many bytes into the first page the area starts */
	46	unsigned int page_offset;
	47	/* IOMMU_READ, IOMMU_WRITE, etc */
	48	int iommu_prot;
	49	bool prevent_access : 1;
	50	unsigned int num_accesses;
	51	};
	52
	53	struct iopt_allowed {
	54	struct interval_tree_node node;
	55	};
	56
	57	struct iopt_reserved {
	58	struct interval_tree_node node;
	59	void *owner;
	60	};
	61
	62	int iopt_area_fill_domains(struct iopt_area area, struct iopt_pages pages);
	63	void iopt_area_unfill_domains(struct iopt_area area, struct iopt_pages pages);
	64
	65	int iopt_area_fill_domain(struct iopt_area area, struct iommu_domain domain);
	66	void iopt_area_unfill_domain(struct iopt_area area, struct iopt_pages pages,
	67	struct iommu_domain *domain);
	68	void iopt_area_unmap_domain(struct iopt_area *area,
	69	struct iommu_domain *domain);
	70
	71	static inline unsigned long iopt_area_index(struct iopt_area *area)
	72	{
	73	return area->pages_node.start;
	74	}
	75
	76	static inline unsigned long iopt_area_last_index(struct iopt_area *area)
	77	{
	78	return area->pages_node.last;
	79	}
	80
	81	static inline unsigned long iopt_area_iova(struct iopt_area *area)
	82	{
	83	return area->node.start;
	84	}
	85
	86	static inline unsigned long iopt_area_last_iova(struct iopt_area *area)
	87	{
	88	return area->node.last;
	89	}
	90
	91	static inline size_t iopt_area_length(struct iopt_area *area)
	92	{
	93	return (area->node.last - area->node.start) + 1;
	94	}
	95
	96	/*
	97	* Number of bytes from the start of the iopt_pages that the iova begins.
	98	* iopt_area_start_byte() / PAGE_SIZE encodes the starting page index
	99	* iopt_area_start_byte() % PAGE_SIZE encodes the offset within that page
	100	*/
	101	static inline unsigned long iopt_area_start_byte(struct iopt_area *area,
	102	unsigned long iova)
	103	{
	104	if (IS_ENABLED(CONFIG_IOMMUFD_TEST))
	105	WARN_ON(iova < iopt_area_iova(area) \|\|
	106	iova > iopt_area_last_iova(area));
	107	return (iova - iopt_area_iova(area)) + area->page_offset +
	108	iopt_area_index(area) * PAGE_SIZE;
	109	}
	110
	111	static inline unsigned long iopt_area_iova_to_index(struct iopt_area *area,
	112	unsigned long iova)
	113	{
	114	return iopt_area_start_byte(area, iova) / PAGE_SIZE;
	115	}
	116
	117	#define __make_iopt_iter(name) \
	118	static inline struct iopt_##name *iopt_##name##_iter_first( \
	119	struct io_pagetable *iopt, unsigned long start, \
	120	unsigned long last) \
	121	{ \
	122	struct interval_tree_node *node; \
	123	\
	124	lockdep_assert_held(&iopt->iova_rwsem); \
	125	node = interval_tree_iter_first(&iopt->name##_itree, start, \
	126	last); \
	127	if (!node) \
	128	return NULL; \
	129	return container_of(node, struct iopt_##name, node); \
	130	} \
	131	static inline struct iopt_##name *iopt_##name##_iter_next( \
	132	struct iopt_##name *last_node, unsigned long start, \
	133	unsigned long last) \
	134	{ \
	135	struct interval_tree_node *node; \
	136	\
	137	node = interval_tree_iter_next(&last_node->node, start, last); \
	138	if (!node) \
	139	return NULL; \
	140	return container_of(node, struct iopt_##name, node); \
	141	}
	142
	143	__make_iopt_iter(area)
	144	__make_iopt_iter(allowed)
	145	__make_iopt_iter(reserved)
	146
	147	struct iopt_area_contig_iter {
	148	unsigned long cur_iova;
	149	unsigned long last_iova;
	150	struct iopt_area *area;
	151	};
	152	struct iopt_area iopt_area_contig_init(struct iopt_area_contig_iter iter,
	153	struct io_pagetable *iopt,
	154	unsigned long iova,
	155	unsigned long last_iova);
	156	struct iopt_area iopt_area_contig_next(struct iopt_area_contig_iter iter);
	157
	158	static inline bool iopt_area_contig_done(struct iopt_area_contig_iter *iter)
	159	{
	160	return iter->area && iter->last_iova <= iopt_area_last_iova(iter->area);
	161	}
	162
	163	/*
	164	* Iterate over a contiguous list of areas that span the iova,last_iova range.
	165	* The caller must check iopt_area_contig_done() after the loop to see if
	166	* contiguous areas existed.
	167	*/
	168	#define iopt_for_each_contig_area(iter, area, iopt, iova, last_iova) \
	169	for (area = iopt_area_contig_init(iter, iopt, iova, last_iova); area; \
	170	area = iopt_area_contig_next(iter))
	171
	172	enum {
	173	IOPT_PAGES_ACCOUNT_NONE = 0,
	174	IOPT_PAGES_ACCOUNT_USER = 1,
	175	IOPT_PAGES_ACCOUNT_MM = 2,
	176	};
	177
	178	/*
	179	* This holds a pinned page list for multiple areas of IO address space. The
	180	* pages always originate from a linear chunk of userspace VA. Multiple
	181	* io_pagetable's, through their iopt_area's, can share a single iopt_pages
	182	* which avoids multi-pinning and double accounting of page consumption.
	183	*
	184	* indexes in this structure are measured in PAGE_SIZE units, are 0 based from
	185	* the start of the uptr and extend to npages. pages are pinned dynamically
	186	* according to the intervals in the access_itree and domains_itree, npinned
	187	* records the current number of pages pinned.
	188	*/
	189	struct iopt_pages {
	190	struct kref kref;
	191	struct mutex mutex;
	192	size_t npages;
	193	size_t npinned;
	194	size_t last_npinned;
	195	struct task_struct *source_task;
	196	struct mm_struct *source_mm;
	197	struct user_struct *source_user;
	198	void __user *uptr;
	199	bool writable:1;
	200	u8 account_mode;
	201
	202	struct xarray pinned_pfns;
	203	/* Of iopt_pages_access::node */
	204	struct rb_root_cached access_itree;
	205	/* Of iopt_area::pages_node */
	206	struct rb_root_cached domains_itree;
	207	};
	208
	209	struct iopt_pages iopt_alloc_pages(void __user uptr, unsigned long length,
	210	bool writable);
	211	void iopt_release_pages(struct kref *kref);
	212	static inline void iopt_put_pages(struct iopt_pages *pages)
	213	{
	214	kref_put(&pages->kref, iopt_release_pages);
	215	}
	216
	217	void iopt_pages_fill_from_xarray(struct iopt_pages *pages, unsigned long start,
	218	unsigned long last, struct page **out_pages);
	219	int iopt_pages_fill_xarray(struct iopt_pages *pages, unsigned long start,
	220	unsigned long last, struct page **out_pages);
	221	void iopt_pages_unfill_xarray(struct iopt_pages *pages, unsigned long start,
	222	unsigned long last);
	223
	224	int iopt_area_add_access(struct iopt_area *area, unsigned long start,
	225	unsigned long last, struct page **out_pages,
	226	unsigned int flags);
	227	void iopt_area_remove_access(struct iopt_area *area, unsigned long start,
	228	unsigned long last);
	229	int iopt_pages_rw_access(struct iopt_pages *pages, unsigned long start_byte,
	230	void *data, unsigned long length, unsigned int flags);
	231
	232	/*
	233	* Each interval represents an active iopt_access_pages(), it acts as an
	234	* interval lock that keeps the PFNs pinned and stored in the xarray.
	235	*/
	236	struct iopt_pages_access {
	237	struct interval_tree_node node;
	238	unsigned int users;
	239	};
	240
	241	#endif

drivers/iommu/iommufd/ioas.c+398−0

@@ -0,0 +1,398 @@
	1	// SPDX-License-Identifier: GPL-2.0-only
	2	/*
	3	* Copyright (c) 2021-2022, NVIDIA CORPORATION & AFFILIATES
	4	*/
	5	#include <linux/interval_tree.h>
	6	#include <linux/iommufd.h>
	7	#include <linux/iommu.h>
	8	#include <uapi/linux/iommufd.h>
	9
	10	#include "io_pagetable.h"
	11
	12	void iommufd_ioas_destroy(struct iommufd_object *obj)
	13	{
	14	struct iommufd_ioas *ioas = container_of(obj, struct iommufd_ioas, obj);
	15	int rc;
	16
	17	rc = iopt_unmap_all(&ioas->iopt, NULL);
	18	WARN_ON(rc && rc != -ENOENT);
	19	iopt_destroy_table(&ioas->iopt);
	20	mutex_destroy(&ioas->mutex);
	21	}
	22
	23	struct iommufd_ioas iommufd_ioas_alloc(struct iommufd_ctx ictx)
	24	{
	25	struct iommufd_ioas *ioas;
	26
	27	ioas = iommufd_object_alloc(ictx, ioas, IOMMUFD_OBJ_IOAS);
	28	if (IS_ERR(ioas))
	29	return ioas;
	30
	31	iopt_init_table(&ioas->iopt);
	32	INIT_LIST_HEAD(&ioas->hwpt_list);
	33	mutex_init(&ioas->mutex);
	34	return ioas;
	35	}
	36
	37	int iommufd_ioas_alloc_ioctl(struct iommufd_ucmd *ucmd)
	38	{
	39	struct iommu_ioas_alloc *cmd = ucmd->cmd;
	40	struct iommufd_ioas *ioas;
	41	int rc;
	42
	43	if (cmd->flags)
	44	return -EOPNOTSUPP;
	45
	46	ioas = iommufd_ioas_alloc(ucmd->ictx);
	47	if (IS_ERR(ioas))
	48	return PTR_ERR(ioas);
	49
	50	cmd->out_ioas_id = ioas->obj.id;
	51	rc = iommufd_ucmd_respond(ucmd, sizeof(*cmd));
	52	if (rc)
	53	goto out_table;
	54	iommufd_object_finalize(ucmd->ictx, &ioas->obj);
	55	return 0;
	56
	57	out_table:
	58	iommufd_object_abort_and_destroy(ucmd->ictx, &ioas->obj);
	59	return rc;
	60	}
	61
	62	int iommufd_ioas_iova_ranges(struct iommufd_ucmd *ucmd)
	63	{
	64	struct iommu_iova_range __user *ranges;
	65	struct iommu_ioas_iova_ranges *cmd = ucmd->cmd;
	66	struct iommufd_ioas *ioas;
	67	struct interval_tree_span_iter span;
	68	u32 max_iovas;
	69	int rc;
	70
	71	if (cmd->__reserved)
	72	return -EOPNOTSUPP;
	73
	74	ioas = iommufd_get_ioas(ucmd, cmd->ioas_id);
	75	if (IS_ERR(ioas))
	76	return PTR_ERR(ioas);
	77
	78	down_read(&ioas->iopt.iova_rwsem);
	79	max_iovas = cmd->num_iovas;
	80	ranges = u64_to_user_ptr(cmd->allowed_iovas);
	81	cmd->num_iovas = 0;
	82	cmd->out_iova_alignment = ioas->iopt.iova_alignment;
	83	interval_tree_for_each_span(&span, &ioas->iopt.reserved_itree, 0,
	84	ULONG_MAX) {
	85	if (!span.is_hole)
	86	continue;
	87	if (cmd->num_iovas < max_iovas) {
	88	struct iommu_iova_range elm = {
	89	.start = span.start_hole,
	90	.last = span.last_hole,
	91	};
	92
	93	if (copy_to_user(&ranges[cmd->num_iovas], &elm,
	94	sizeof(elm))) {
	95	rc = -EFAULT;
	96	goto out_put;
	97	}
	98	}
	99	cmd->num_iovas++;
	100	}
	101	rc = iommufd_ucmd_respond(ucmd, sizeof(*cmd));
	102	if (rc)
	103	goto out_put;
	104	if (cmd->num_iovas > max_iovas)
	105	rc = -EMSGSIZE;
	106	out_put:
	107	up_read(&ioas->iopt.iova_rwsem);
	108	iommufd_put_object(&ioas->obj);
	109	return rc;
	110	}
	111
	112	static int iommufd_ioas_load_iovas(struct rb_root_cached *itree,
	113	struct iommu_iova_range __user *ranges,
	114	u32 num)
	115	{
	116	u32 i;
	117
	118	for (i = 0; i != num; i++) {
	119	struct iommu_iova_range range;
	120	struct iopt_allowed *allowed;
	121
	122	if (copy_from_user(&range, ranges + i, sizeof(range)))
	123	return -EFAULT;
	124
	125	if (range.start >= range.last)
	126	return -EINVAL;
	127
	128	if (interval_tree_iter_first(itree, range.start, range.last))
	129	return -EINVAL;
	130
	131	allowed = kzalloc(sizeof(*allowed), GFP_KERNEL_ACCOUNT);
	132	if (!allowed)
	133	return -ENOMEM;
	134	allowed->node.start = range.start;
	135	allowed->node.last = range.last;
	136
	137	interval_tree_insert(&allowed->node, itree);
	138	}
	139	return 0;
	140	}
	141
	142	int iommufd_ioas_allow_iovas(struct iommufd_ucmd *ucmd)
	143	{
	144	struct iommu_ioas_allow_iovas *cmd = ucmd->cmd;
	145	struct rb_root_cached allowed_iova = RB_ROOT_CACHED;
	146	struct interval_tree_node *node;
	147	struct iommufd_ioas *ioas;
	148	struct io_pagetable *iopt;
	149	int rc = 0;
	150
	151	if (cmd->__reserved)
	152	return -EOPNOTSUPP;
	153
	154	ioas = iommufd_get_ioas(ucmd, cmd->ioas_id);
	155	if (IS_ERR(ioas))
	156	return PTR_ERR(ioas);
	157	iopt = &ioas->iopt;
	158
	159	rc = iommufd_ioas_load_iovas(&allowed_iova,
	160	u64_to_user_ptr(cmd->allowed_iovas),
	161	cmd->num_iovas);
	162	if (rc)
	163	goto out_free;
	164
	165	/*
	166	* We want the allowed tree update to be atomic, so we have to keep the
	167	* original nodes around, and keep track of the new nodes as we allocate
	168	* memory for them. The simplest solution is to have a new/old tree and
	169	* then swap new for old. On success we free the old tree, on failure we
	170	* free the new tree.
	171	*/
	172	rc = iopt_set_allow_iova(iopt, &allowed_iova);
	173	out_free:
	174	while ((node = interval_tree_iter_first(&allowed_iova, 0, ULONG_MAX))) {
	175	interval_tree_remove(node, &allowed_iova);
	176	kfree(container_of(node, struct iopt_allowed, node));
	177	}
	178	iommufd_put_object(&ioas->obj);
	179	return rc;
	180	}
	181
	182	static int conv_iommu_prot(u32 map_flags)
	183	{
	184	/*
	185	* We provide no manual cache coherency ioctls to userspace and most
	186	* architectures make the CPU ops for cache flushing privileged.
	187	* Therefore we require the underlying IOMMU to support CPU coherent
	188	* operation. Support for IOMMU_CACHE is enforced by the
	189	* IOMMU_CAP_CACHE_COHERENCY test during bind.
	190	*/
	191	int iommu_prot = IOMMU_CACHE;
	192
	193	if (map_flags & IOMMU_IOAS_MAP_WRITEABLE)
	194	iommu_prot \|= IOMMU_WRITE;
	195	if (map_flags & IOMMU_IOAS_MAP_READABLE)
	196	iommu_prot \|= IOMMU_READ;
	197	return iommu_prot;
	198	}
	199
	200	int iommufd_ioas_map(struct iommufd_ucmd *ucmd)
	201	{
	202	struct iommu_ioas_map *cmd = ucmd->cmd;
	203	unsigned long iova = cmd->iova;
	204	struct iommufd_ioas *ioas;
	205	unsigned int flags = 0;
	206	int rc;
	207
	208	if ((cmd->flags &
	209	~(IOMMU_IOAS_MAP_FIXED_IOVA \| IOMMU_IOAS_MAP_WRITEABLE \|
	210	IOMMU_IOAS_MAP_READABLE)) \|\|
	211	cmd->__reserved)
	212	return -EOPNOTSUPP;
	213	if (cmd->iova >= ULONG_MAX \|\| cmd->length >= ULONG_MAX)
	214	return -EOVERFLOW;
	215
	216	ioas = iommufd_get_ioas(ucmd, cmd->ioas_id);
	217	if (IS_ERR(ioas))
	218	return PTR_ERR(ioas);
	219
	220	if (!(cmd->flags & IOMMU_IOAS_MAP_FIXED_IOVA))
	221	flags = IOPT_ALLOC_IOVA;
	222	rc = iopt_map_user_pages(ucmd->ictx, &ioas->iopt, &iova,
	223	u64_to_user_ptr(cmd->user_va), cmd->length,
	224	conv_iommu_prot(cmd->flags), flags);
	225	if (rc)
	226	goto out_put;
	227
	228	cmd->iova = iova;
	229	rc = iommufd_ucmd_respond(ucmd, sizeof(*cmd));
	230	out_put:
	231	iommufd_put_object(&ioas->obj);
	232	return rc;
	233	}
	234
	235	int iommufd_ioas_copy(struct iommufd_ucmd *ucmd)
	236	{
	237	struct iommu_ioas_copy *cmd = ucmd->cmd;
	238	struct iommufd_ioas *src_ioas;
	239	struct iommufd_ioas *dst_ioas;
	240	unsigned int flags = 0;
	241	LIST_HEAD(pages_list);
	242	unsigned long iova;
	243	int rc;
	244
	245	iommufd_test_syz_conv_iova_id(ucmd, cmd->src_ioas_id, &cmd->src_iova,
	246	&cmd->flags);
	247
	248	if ((cmd->flags &
	249	~(IOMMU_IOAS_MAP_FIXED_IOVA \| IOMMU_IOAS_MAP_WRITEABLE \|
	250	IOMMU_IOAS_MAP_READABLE)))
	251	return -EOPNOTSUPP;
	252	if (cmd->length >= ULONG_MAX \|\| cmd->src_iova >= ULONG_MAX \|\|
	253	cmd->dst_iova >= ULONG_MAX)
	254	return -EOVERFLOW;
	255
	256	src_ioas = iommufd_get_ioas(ucmd, cmd->src_ioas_id);
	257	if (IS_ERR(src_ioas))
	258	return PTR_ERR(src_ioas);
	259	rc = iopt_get_pages(&src_ioas->iopt, cmd->src_iova, cmd->length,
	260	&pages_list);
	261	iommufd_put_object(&src_ioas->obj);
	262	if (rc)
	263	return rc;
	264
	265	dst_ioas = iommufd_get_ioas(ucmd, cmd->dst_ioas_id);
	266	if (IS_ERR(dst_ioas)) {
	267	rc = PTR_ERR(dst_ioas);
	268	goto out_pages;
	269	}
	270
	271	if (!(cmd->flags & IOMMU_IOAS_MAP_FIXED_IOVA))
	272	flags = IOPT_ALLOC_IOVA;
	273	iova = cmd->dst_iova;
	274	rc = iopt_map_pages(&dst_ioas->iopt, &pages_list, cmd->length, &iova,
	275	conv_iommu_prot(cmd->flags), flags);
	276	if (rc)
	277	goto out_put_dst;
	278
	279	cmd->dst_iova = iova;
	280	rc = iommufd_ucmd_respond(ucmd, sizeof(*cmd));
	281	out_put_dst:
	282	iommufd_put_object(&dst_ioas->obj);
	283	out_pages:
	284	iopt_free_pages_list(&pages_list);
	285	return rc;
	286	}
	287
	288	int iommufd_ioas_unmap(struct iommufd_ucmd *ucmd)
	289	{
	290	struct iommu_ioas_unmap *cmd = ucmd->cmd;
	291	struct iommufd_ioas *ioas;
	292	unsigned long unmapped = 0;
	293	int rc;
	294
	295	ioas = iommufd_get_ioas(ucmd, cmd->ioas_id);
	296	if (IS_ERR(ioas))
	297	return PTR_ERR(ioas);
	298
	299	if (cmd->iova == 0 && cmd->length == U64_MAX) {
	300	rc = iopt_unmap_all(&ioas->iopt, &unmapped);
	301	if (rc)
	302	goto out_put;
	303	} else {
	304	if (cmd->iova >= ULONG_MAX \|\| cmd->length >= ULONG_MAX) {
	305	rc = -EOVERFLOW;
	306	goto out_put;
	307	}
	308	rc = iopt_unmap_iova(&ioas->iopt, cmd->iova, cmd->length,
	309	&unmapped);
	310	if (rc)
	311	goto out_put;
	312	}
	313
	314	cmd->length = unmapped;
	315	rc = iommufd_ucmd_respond(ucmd, sizeof(*cmd));
	316
	317	out_put:
	318	iommufd_put_object(&ioas->obj);
	319	return rc;
	320	}
	321
	322	int iommufd_option_rlimit_mode(struct iommu_option *cmd,
	323	struct iommufd_ctx *ictx)
	324	{
	325	if (cmd->object_id)
	326	return -EOPNOTSUPP;
	327
	328	if (cmd->op == IOMMU_OPTION_OP_GET) {
	329	cmd->val64 = ictx->account_mode == IOPT_PAGES_ACCOUNT_MM;
	330	return 0;
	331	}
	332	if (cmd->op == IOMMU_OPTION_OP_SET) {
	333	int rc = 0;
	334
	335	if (!capable(CAP_SYS_RESOURCE))
	336	return -EPERM;
	337
	338	xa_lock(&ictx->objects);
	339	if (!xa_empty(&ictx->objects)) {
	340	rc = -EBUSY;
	341	} else {
	342	if (cmd->val64 == 0)
	343	ictx->account_mode = IOPT_PAGES_ACCOUNT_USER;
	344	else if (cmd->val64 == 1)
	345	ictx->account_mode = IOPT_PAGES_ACCOUNT_MM;
	346	else
	347	rc = -EINVAL;
	348	}
	349	xa_unlock(&ictx->objects);
	350
	351	return rc;
	352	}
	353	return -EOPNOTSUPP;
	354	}
	355
	356	static int iommufd_ioas_option_huge_pages(struct iommu_option *cmd,
	357	struct iommufd_ioas *ioas)
	358	{
	359	if (cmd->op == IOMMU_OPTION_OP_GET) {
	360	cmd->val64 = !ioas->iopt.disable_large_pages;
	361	return 0;
	362	}
	363	if (cmd->op == IOMMU_OPTION_OP_SET) {
	364	if (cmd->val64 == 0)
	365	return iopt_disable_large_pages(&ioas->iopt);
	366	if (cmd->val64 == 1) {
	367	iopt_enable_large_pages(&ioas->iopt);
	368	return 0;
	369	}
	370	return -EINVAL;
	371	}
	372	return -EOPNOTSUPP;
	373	}
	374
	375	int iommufd_ioas_option(struct iommufd_ucmd *ucmd)
	376	{
	377	struct iommu_option *cmd = ucmd->cmd;
	378	struct iommufd_ioas *ioas;
	379	int rc = 0;
	380
	381	if (cmd->__reserved)
	382	return -EOPNOTSUPP;
	383
	384	ioas = iommufd_get_ioas(ucmd, cmd->object_id);
	385	if (IS_ERR(ioas))
	386	return PTR_ERR(ioas);
	387
	388	switch (cmd->option_id) {
	389	case IOMMU_OPTION_HUGE_PAGES:
	390	rc = iommufd_ioas_option_huge_pages(cmd, ioas);
	391	break;
	392	default:
	393	rc = -EOPNOTSUPP;
	394	}
	395
	396	iommufd_put_object(&ioas->obj);
	397	return rc;
	398	}

drivers/iommu/iommufd/iommufd_private.h+307−0

@@ -0,0 +1,307 @@
	1	/* SPDX-License-Identifier: GPL-2.0-only */
	2	/* Copyright (c) 2021-2022, NVIDIA CORPORATION & AFFILIATES
	3	*/
	4	#ifndef __IOMMUFD_PRIVATE_H
	5	#define __IOMMUFD_PRIVATE_H
	6
	7	#include <linux/rwsem.h>
	8	#include <linux/xarray.h>
	9	#include <linux/refcount.h>
	10	#include <linux/uaccess.h>
	11
	12	struct iommu_domain;
	13	struct iommu_group;
	14	struct iommu_option;
	15
	16	struct iommufd_ctx {
	17	struct file *file;
	18	struct xarray objects;
	19
	20	u8 account_mode;
	21	struct iommufd_ioas *vfio_ioas;
	22	};
	23
	24	/*
	25	* The IOVA to PFN map. The map automatically copies the PFNs into multiple
	26	* domains and permits sharing of PFNs between io_pagetable instances. This
	27	* supports both a design where IOAS's are 1:1 with a domain (eg because the
	28	* domain is HW customized), or where the IOAS is 1:N with multiple generic
	29	* domains. The io_pagetable holds an interval tree of iopt_areas which point
	30	* to shared iopt_pages which hold the pfns mapped to the page table.
	31	*
	32	* The locking order is domains_rwsem -> iova_rwsem -> pages::mutex
	33	*/
	34	struct io_pagetable {
	35	struct rw_semaphore domains_rwsem;
	36	struct xarray domains;
	37	struct xarray access_list;
	38	unsigned int next_domain_id;
	39
	40	struct rw_semaphore iova_rwsem;
	41	struct rb_root_cached area_itree;
	42	/* IOVA that cannot become reserved, struct iopt_allowed */
	43	struct rb_root_cached allowed_itree;
	44	/* IOVA that cannot be allocated, struct iopt_reserved */
	45	struct rb_root_cached reserved_itree;
	46	u8 disable_large_pages;
	47	unsigned long iova_alignment;
	48	};
	49
	50	void iopt_init_table(struct io_pagetable *iopt);
	51	void iopt_destroy_table(struct io_pagetable *iopt);
	52	int iopt_get_pages(struct io_pagetable *iopt, unsigned long iova,
	53	unsigned long length, struct list_head *pages_list);
	54	void iopt_free_pages_list(struct list_head *pages_list);
	55	enum {
	56	IOPT_ALLOC_IOVA = 1 << 0,
	57	};
	58	int iopt_map_user_pages(struct iommufd_ctx ictx, struct io_pagetable iopt,
	59	unsigned long iova, void __user uptr,
	60	unsigned long length, int iommu_prot,
	61	unsigned int flags);
	62	int iopt_map_pages(struct io_pagetable iopt, struct list_head pages_list,
	63	unsigned long length, unsigned long *dst_iova,
	64	int iommu_prot, unsigned int flags);
	65	int iopt_unmap_iova(struct io_pagetable *iopt, unsigned long iova,
	66	unsigned long length, unsigned long *unmapped);
	67	int iopt_unmap_all(struct io_pagetable iopt, unsigned long unmapped);
	68
	69	void iommufd_access_notify_unmap(struct io_pagetable *iopt, unsigned long iova,
	70	unsigned long length);
	71	int iopt_table_add_domain(struct io_pagetable *iopt,
	72	struct iommu_domain *domain);
	73	void iopt_table_remove_domain(struct io_pagetable *iopt,
	74	struct iommu_domain *domain);
	75	int iopt_table_enforce_group_resv_regions(struct io_pagetable *iopt,
	76	struct device *device,
	77	struct iommu_group *group,
	78	phys_addr_t *sw_msi_start);
	79	int iopt_set_allow_iova(struct io_pagetable *iopt,
	80	struct rb_root_cached *allowed_iova);
	81	int iopt_reserve_iova(struct io_pagetable *iopt, unsigned long start,
	82	unsigned long last, void *owner);
	83	void iopt_remove_reserved_iova(struct io_pagetable iopt, void owner);
	84	int iopt_cut_iova(struct io_pagetable iopt, unsigned long iovas,
	85	size_t num_iovas);
	86	void iopt_enable_large_pages(struct io_pagetable *iopt);
	87	int iopt_disable_large_pages(struct io_pagetable *iopt);
	88
	89	struct iommufd_ucmd {
	90	struct iommufd_ctx *ictx;
	91	void __user *ubuffer;
	92	u32 user_size;
	93	void *cmd;
	94	};
	95
	96	int iommufd_vfio_ioctl(struct iommufd_ctx *ictx, unsigned int cmd,
	97	unsigned long arg);
	98
	99	/* Copy the response in ucmd->cmd back to userspace. */
	100	static inline int iommufd_ucmd_respond(struct iommufd_ucmd *ucmd,
	101	size_t cmd_len)
	102	{
	103	if (copy_to_user(ucmd->ubuffer, ucmd->cmd,
	104	min_t(size_t, ucmd->user_size, cmd_len)))
	105	return -EFAULT;
	106	return 0;
	107	}
	108
	109	enum iommufd_object_type {
	110	IOMMUFD_OBJ_NONE,
	111	IOMMUFD_OBJ_ANY = IOMMUFD_OBJ_NONE,
	112	IOMMUFD_OBJ_DEVICE,
	113	IOMMUFD_OBJ_HW_PAGETABLE,
	114	IOMMUFD_OBJ_IOAS,
	115	IOMMUFD_OBJ_ACCESS,
	116	#ifdef CONFIG_IOMMUFD_TEST
	117	IOMMUFD_OBJ_SELFTEST,
	118	#endif
	119	};
	120
	121	/* Base struct for all objects with a userspace ID handle. */
	122	struct iommufd_object {
	123	struct rw_semaphore destroy_rwsem;
	124	refcount_t users;
	125	enum iommufd_object_type type;
	126	unsigned int id;
	127	};
	128
	129	static inline bool iommufd_lock_obj(struct iommufd_object *obj)
	130	{
	131	if (!down_read_trylock(&obj->destroy_rwsem))
	132	return false;
	133	if (!refcount_inc_not_zero(&obj->users)) {
	134	up_read(&obj->destroy_rwsem);
	135	return false;
	136	}
	137	return true;
	138	}
	139
	140	struct iommufd_object iommufd_get_object(struct iommufd_ctx ictx, u32 id,
	141	enum iommufd_object_type type);
	142	static inline void iommufd_put_object(struct iommufd_object *obj)
	143	{
	144	refcount_dec(&obj->users);
	145	up_read(&obj->destroy_rwsem);
	146	}
	147
	148	/**
	149	* iommufd_ref_to_users() - Switch from destroy_rwsem to users refcount
	150	* protection
	151	* @obj - Object to release
	152	*
	153	* Objects have two refcount protections (destroy_rwsem and the refcount_t
	154	* users). Holding either of these will prevent the object from being destroyed.
	155	*
	156	* Depending on the use case, one protection or the other is appropriate. In
	157	* most cases references are being protected by the destroy_rwsem. This allows
	158	* orderly destruction of the object because iommufd_object_destroy_user() will
	159	* wait for it to become unlocked. However, as a rwsem, it cannot be held across
	160	* a system call return. So cases that have longer term needs must switch
	161	* to the weaker users refcount_t.
	162	*
	163	* With users protection iommufd_object_destroy_user() will return false,
	164	* refusing to destroy the object, causing -EBUSY to userspace.
	165	*/
	166	static inline void iommufd_ref_to_users(struct iommufd_object *obj)
	167	{
	168	up_read(&obj->destroy_rwsem);
	169	/* iommufd_lock_obj() obtains users as well */
	170	}
	171	void iommufd_object_abort(struct iommufd_ctx ictx, struct iommufd_object obj);
	172	void iommufd_object_abort_and_destroy(struct iommufd_ctx *ictx,
	173	struct iommufd_object *obj);
	174	void iommufd_object_finalize(struct iommufd_ctx *ictx,
	175	struct iommufd_object *obj);
	176	bool iommufd_object_destroy_user(struct iommufd_ctx *ictx,
	177	struct iommufd_object *obj);
	178	struct iommufd_object _iommufd_object_alloc(struct iommufd_ctx ictx,
	179	size_t size,
	180	enum iommufd_object_type type);
	181
	182	#define iommufd_object_alloc(ictx, ptr, type) \
	183	container_of(_iommufd_object_alloc( \
	184	ictx, \
	185	sizeof(*(ptr)) + BUILD_BUG_ON_ZERO( \
	186	offsetof(typeof(*(ptr)), \
	187	obj) != 0), \
	188	type), \
	189	typeof(*(ptr)), obj)
	190
	191	/*
	192	* The IO Address Space (IOAS) pagetable is a virtual page table backed by the
	193	* io_pagetable object. It is a user controlled mapping of IOVA -> PFNs. The
	194	* mapping is copied into all of the associated domains and made available to
	195	* in-kernel users.
	196	*
	197	* Every iommu_domain that is created is wrapped in a iommufd_hw_pagetable
	198	* object. When we go to attach a device to an IOAS we need to get an
	199	* iommu_domain and wrapping iommufd_hw_pagetable for it.
	200	*
	201	* An iommu_domain & iommfd_hw_pagetable will be automatically selected
	202	* for a device based on the hwpt_list. If no suitable iommu_domain
	203	* is found a new iommu_domain will be created.
	204	*/
	205	struct iommufd_ioas {
	206	struct iommufd_object obj;
	207	struct io_pagetable iopt;
	208	struct mutex mutex;
	209	struct list_head hwpt_list;
	210	};
	211
	212	static inline struct iommufd_ioas iommufd_get_ioas(struct iommufd_ucmd ucmd,
	213	u32 id)
	214	{
	215	return container_of(iommufd_get_object(ucmd->ictx, id,
	216	IOMMUFD_OBJ_IOAS),
	217	struct iommufd_ioas, obj);
	218	}
	219
	220	struct iommufd_ioas iommufd_ioas_alloc(struct iommufd_ctx ictx);
	221	int iommufd_ioas_alloc_ioctl(struct iommufd_ucmd *ucmd);
	222	void iommufd_ioas_destroy(struct iommufd_object *obj);
	223	int iommufd_ioas_iova_ranges(struct iommufd_ucmd *ucmd);
	224	int iommufd_ioas_allow_iovas(struct iommufd_ucmd *ucmd);
	225	int iommufd_ioas_map(struct iommufd_ucmd *ucmd);
	226	int iommufd_ioas_copy(struct iommufd_ucmd *ucmd);
	227	int iommufd_ioas_unmap(struct iommufd_ucmd *ucmd);
	228	int iommufd_ioas_option(struct iommufd_ucmd *ucmd);
	229	int iommufd_option_rlimit_mode(struct iommu_option *cmd,
	230	struct iommufd_ctx *ictx);
	231
	232	int iommufd_vfio_ioas(struct iommufd_ucmd *ucmd);
	233
	234	/*
	235	* A HW pagetable is called an iommu_domain inside the kernel. This user object
	236	* allows directly creating and inspecting the domains. Domains that have kernel
	237	* owned page tables will be associated with an iommufd_ioas that provides the
	238	* IOVA to PFN map.
	239	*/
	240	struct iommufd_hw_pagetable {
	241	struct iommufd_object obj;
	242	struct iommufd_ioas *ioas;
	243	struct iommu_domain *domain;
	244	bool auto_domain : 1;
	245	bool enforce_cache_coherency : 1;
	246	bool msi_cookie : 1;
	247	/* Head at iommufd_ioas::hwpt_list */
	248	struct list_head hwpt_item;
	249	struct mutex devices_lock;
	250	struct list_head devices;
	251	};
	252
	253	struct iommufd_hw_pagetable *
	254	iommufd_hw_pagetable_alloc(struct iommufd_ctx ictx, struct iommufd_ioas ioas,
	255	struct device *dev);
	256	void iommufd_hw_pagetable_destroy(struct iommufd_object *obj);
	257
	258	void iommufd_device_destroy(struct iommufd_object *obj);
	259
	260	struct iommufd_access {
	261	struct iommufd_object obj;
	262	struct iommufd_ctx *ictx;
	263	struct iommufd_ioas *ioas;
	264	const struct iommufd_access_ops *ops;
	265	void *data;
	266	unsigned long iova_alignment;
	267	u32 iopt_access_list_id;
	268	};
	269
	270	int iopt_add_access(struct io_pagetable iopt, struct iommufd_access access);
	271	void iopt_remove_access(struct io_pagetable *iopt,
	272	struct iommufd_access *access);
	273	void iommufd_access_destroy_object(struct iommufd_object *obj);
	274
	275	#ifdef CONFIG_IOMMUFD_TEST
	276	struct iommufd_hw_pagetable *
	277	iommufd_device_selftest_attach(struct iommufd_ctx *ictx,
	278	struct iommufd_ioas *ioas,
	279	struct device *mock_dev);
	280	void iommufd_device_selftest_detach(struct iommufd_ctx *ictx,
	281	struct iommufd_hw_pagetable *hwpt);
	282	int iommufd_test(struct iommufd_ucmd *ucmd);
	283	void iommufd_selftest_destroy(struct iommufd_object *obj);
	284	extern size_t iommufd_test_memory_limit;
	285	void iommufd_test_syz_conv_iova_id(struct iommufd_ucmd *ucmd,
	286	unsigned int ioas_id, u64 iova, u32 flags);
	287	bool iommufd_should_fail(void);
	288	void __init iommufd_test_init(void);
	289	void iommufd_test_exit(void);
	290	#else
	291	static inline void iommufd_test_syz_conv_iova_id(struct iommufd_ucmd *ucmd,
	292	unsigned int ioas_id,
	293	u64 iova, u32 flags)
	294	{
	295	}
	296	static inline bool iommufd_should_fail(void)
	297	{
	298	return false;
	299	}
	300	static inline void __init iommufd_test_init(void)
	301	{
	302	}
	303	static inline void iommufd_test_exit(void)
	304	{
	305	}
	306	#endif
	307	#endif

drivers/iommu/iommufd/iommufd_test.h+93−0

@@ -0,0 +1,93 @@
	1	/* SPDX-License-Identifier: GPL-2.0 */
	2	/* Copyright (c) 2021-2022, NVIDIA CORPORATION & AFFILIATES.
	3	*/
	4	#ifndef _UAPI_IOMMUFD_TEST_H
	5	#define _UAPI_IOMMUFD_TEST_H
	6
	7	#include <linux/types.h>
	8	#include <linux/iommufd.h>
	9
	10	enum {
	11	IOMMU_TEST_OP_ADD_RESERVED = 1,
	12	IOMMU_TEST_OP_MOCK_DOMAIN,
	13	IOMMU_TEST_OP_MD_CHECK_MAP,
	14	IOMMU_TEST_OP_MD_CHECK_REFS,
	15	IOMMU_TEST_OP_CREATE_ACCESS,
	16	IOMMU_TEST_OP_DESTROY_ACCESS_PAGES,
	17	IOMMU_TEST_OP_ACCESS_PAGES,
	18	IOMMU_TEST_OP_ACCESS_RW,
	19	IOMMU_TEST_OP_SET_TEMP_MEMORY_LIMIT,
	20	};
	21
	22	enum {
	23	MOCK_APERTURE_START = 1UL << 24,
	24	MOCK_APERTURE_LAST = (1UL << 31) - 1,
	25	};
	26
	27	enum {
	28	MOCK_FLAGS_ACCESS_WRITE = 1 << 0,
	29	MOCK_FLAGS_ACCESS_SYZ = 1 << 16,
	30	};
	31
	32	enum {
	33	MOCK_ACCESS_RW_WRITE = 1 << 0,
	34	MOCK_ACCESS_RW_SLOW_PATH = 1 << 2,
	35	};
	36
	37	enum {
	38	MOCK_FLAGS_ACCESS_CREATE_NEEDS_PIN_PAGES = 1 << 0,
	39	};
	40
	41	struct iommu_test_cmd {
	42	__u32 size;
	43	__u32 op;
	44	__u32 id;
	45	__u32 __reserved;
	46	union {
	47	struct {
	48	__aligned_u64 start;
	49	__aligned_u64 length;
	50	} add_reserved;
	51	struct {
	52	__u32 out_device_id;
	53	__u32 out_hwpt_id;
	54	} mock_domain;
	55	struct {
	56	__aligned_u64 iova;
	57	__aligned_u64 length;
	58	__aligned_u64 uptr;
	59	} check_map;
	60	struct {
	61	__aligned_u64 length;
	62	__aligned_u64 uptr;
	63	__u32 refs;
	64	} check_refs;
	65	struct {
	66	__u32 out_access_fd;
	67	__u32 flags;
	68	} create_access;
	69	struct {
	70	__u32 access_pages_id;
	71	} destroy_access_pages;
	72	struct {
	73	__u32 flags;
	74	__u32 out_access_pages_id;
	75	__aligned_u64 iova;
	76	__aligned_u64 length;
	77	__aligned_u64 uptr;
	78	} access_pages;
	79	struct {
	80	__aligned_u64 iova;
	81	__aligned_u64 length;
	82	__aligned_u64 uptr;
	83	__u32 flags;
	84	} access_rw;
	85	struct {
	86	__u32 limit;
	87	} memory_limit;
	88	};
	89	__u32 last;
	90	};
	91	#define IOMMU_TEST_CMD _IO(IOMMUFD_TYPE, IOMMUFD_CMD_BASE + 32)
	92
	93	#endif

drivers/iommu/iommufd/main.c+460−0

@@ -0,0 +1,460 @@
	1	// SPDX-License-Identifier: GPL-2.0-only
	2	/* Copyright (C) 2021 Intel Corporation
	3	* Copyright (c) 2021-2022, NVIDIA CORPORATION & AFFILIATES
	4	*
	5	* iommufd provides control over the IOMMU HW objects created by IOMMU kernel
	6	* drivers. IOMMU HW objects revolve around IO page tables that map incoming DMA
	7	* addresses (IOVA) to CPU addresses.
	8	*/
	9	#define pr_fmt(fmt) "iommufd: " fmt
	10
	11	#include <linux/file.h>
	12	#include <linux/fs.h>
	13	#include <linux/module.h>
	14	#include <linux/slab.h>
	15	#include <linux/miscdevice.h>
	16	#include <linux/mutex.h>
	17	#include <linux/bug.h>
	18	#include <uapi/linux/iommufd.h>
	19	#include <linux/iommufd.h>
	20
	21	#include "io_pagetable.h"
	22	#include "iommufd_private.h"
	23	#include "iommufd_test.h"
	24
	25	struct iommufd_object_ops {
	26	void (destroy)(struct iommufd_object obj);
	27	};
	28	static const struct iommufd_object_ops iommufd_object_ops[];
	29	static struct miscdevice vfio_misc_dev;
	30
	31	struct iommufd_object _iommufd_object_alloc(struct iommufd_ctx ictx,
	32	size_t size,
	33	enum iommufd_object_type type)
	34	{
	35	struct iommufd_object *obj;
	36	int rc;
	37
	38	obj = kzalloc(size, GFP_KERNEL_ACCOUNT);
	39	if (!obj)
	40	return ERR_PTR(-ENOMEM);
	41	obj->type = type;
	42	init_rwsem(&obj->destroy_rwsem);
	43	refcount_set(&obj->users, 1);
	44
	45	/*
	46	* Reserve an ID in the xarray but do not publish the pointer yet since
	47	* the caller hasn't initialized it yet. Once the pointer is published
	48	* in the xarray and visible to other threads we can't reliably destroy
	49	* it anymore, so the caller must complete all errorable operations
	50	* before calling iommufd_object_finalize().
	51	*/
	52	rc = xa_alloc(&ictx->objects, &obj->id, XA_ZERO_ENTRY,
	53	xa_limit_32b, GFP_KERNEL_ACCOUNT);
	54	if (rc)
	55	goto out_free;
	56	return obj;
	57	out_free:
	58	kfree(obj);
	59	return ERR_PTR(rc);
	60	}
	61
	62	/*
	63	* Allow concurrent access to the object.
	64	*
	65	* Once another thread can see the object pointer it can prevent object
	66	* destruction. Expect for special kernel-only objects there is no in-kernel way
	67	* to reliably destroy a single object. Thus all APIs that are creating objects
	68	* must use iommufd_object_abort() to handle their errors and only call
	69	* iommufd_object_finalize() once object creation cannot fail.
	70	*/
	71	void iommufd_object_finalize(struct iommufd_ctx *ictx,
	72	struct iommufd_object *obj)
	73	{
	74	void *old;
	75
	76	old = xa_store(&ictx->objects, obj->id, obj, GFP_KERNEL);
	77	/* obj->id was returned from xa_alloc() so the xa_store() cannot fail */
	78	WARN_ON(old);
	79	}
	80
	81	/* Undo _iommufd_object_alloc() if iommufd_object_finalize() was not called */
	82	void iommufd_object_abort(struct iommufd_ctx ictx, struct iommufd_object obj)
	83	{
	84	void *old;
	85
	86	old = xa_erase(&ictx->objects, obj->id);
	87	WARN_ON(old);
	88	kfree(obj);
	89	}
	90
	91	/*
	92	* Abort an object that has been fully initialized and needs destroy, but has
	93	* not been finalized.
	94	*/
	95	void iommufd_object_abort_and_destroy(struct iommufd_ctx *ictx,
	96	struct iommufd_object *obj)
	97	{
	98	iommufd_object_ops[obj->type].destroy(obj);
	99	iommufd_object_abort(ictx, obj);
	100	}
	101
	102	struct iommufd_object iommufd_get_object(struct iommufd_ctx ictx, u32 id,
	103	enum iommufd_object_type type)
	104	{
	105	struct iommufd_object *obj;
	106
	107	if (iommufd_should_fail())
	108	return ERR_PTR(-ENOENT);
	109
	110	xa_lock(&ictx->objects);
	111	obj = xa_load(&ictx->objects, id);
	112	if (!obj \|\| (type != IOMMUFD_OBJ_ANY && obj->type != type) \|\|
	113	!iommufd_lock_obj(obj))
	114	obj = ERR_PTR(-ENOENT);
	115	xa_unlock(&ictx->objects);
	116	return obj;
	117	}
	118
	119	/*
	120	* The caller holds a users refcount and wants to destroy the object. Returns
	121	* true if the object was destroyed. In all cases the caller no longer has a
	122	* reference on obj.
	123	*/
	124	bool iommufd_object_destroy_user(struct iommufd_ctx *ictx,
	125	struct iommufd_object *obj)
	126	{
	127	/*
	128	* The purpose of the destroy_rwsem is to ensure deterministic
	129	* destruction of objects used by external drivers and destroyed by this
	130	* function. Any temporary increment of the refcount must hold the read
	131	* side of this, such as during ioctl execution.
	132	*/
	133	down_write(&obj->destroy_rwsem);
	134	xa_lock(&ictx->objects);
	135	refcount_dec(&obj->users);
	136	if (!refcount_dec_if_one(&obj->users)) {
	137	xa_unlock(&ictx->objects);
	138	up_write(&obj->destroy_rwsem);
	139	return false;
	140	}
	141	__xa_erase(&ictx->objects, obj->id);
	142	if (ictx->vfio_ioas && &ictx->vfio_ioas->obj == obj)
	143	ictx->vfio_ioas = NULL;
	144	xa_unlock(&ictx->objects);
	145	up_write(&obj->destroy_rwsem);
	146
	147	iommufd_object_ops[obj->type].destroy(obj);
	148	kfree(obj);
	149	return true;
	150	}
	151
	152	static int iommufd_destroy(struct iommufd_ucmd *ucmd)
	153	{
	154	struct iommu_destroy *cmd = ucmd->cmd;
	155	struct iommufd_object *obj;
	156
	157	obj = iommufd_get_object(ucmd->ictx, cmd->id, IOMMUFD_OBJ_ANY);
	158	if (IS_ERR(obj))
	159	return PTR_ERR(obj);
	160	iommufd_ref_to_users(obj);
	161	/* See iommufd_ref_to_users() */
	162	if (!iommufd_object_destroy_user(ucmd->ictx, obj))
	163	return -EBUSY;
	164	return 0;
	165	}
	166
	167	static int iommufd_fops_open(struct inode inode, struct file filp)
	168	{
	169	struct iommufd_ctx *ictx;
	170
	171	ictx = kzalloc(sizeof(*ictx), GFP_KERNEL_ACCOUNT);
	172	if (!ictx)
	173	return -ENOMEM;
	174
	175	/*
	176	* For compatibility with VFIO when /dev/vfio/vfio is opened we default
	177	* to the same rlimit accounting as vfio uses.
	178	*/
	179	if (IS_ENABLED(CONFIG_IOMMUFD_VFIO_CONTAINER) &&
	180	filp->private_data == &vfio_misc_dev) {
	181	ictx->account_mode = IOPT_PAGES_ACCOUNT_MM;
	182	pr_info_once("IOMMUFD is providing /dev/vfio/vfio, not VFIO.\n");
	183	}
	184
	185	xa_init_flags(&ictx->objects, XA_FLAGS_ALLOC1 \| XA_FLAGS_ACCOUNT);
	186	ictx->file = filp;
	187	filp->private_data = ictx;
	188	return 0;
	189	}
	190
	191	static int iommufd_fops_release(struct inode inode, struct file filp)
	192	{
	193	struct iommufd_ctx *ictx = filp->private_data;
	194	struct iommufd_object *obj;
	195
	196	/*
	197	* The objects in the xarray form a graph of "users" counts, and we have
	198	* to destroy them in a depth first manner. Leaf objects will reduce the
	199	* users count of interior objects when they are destroyed.
	200	*
	201	* Repeatedly destroying all the "1 users" leaf objects will progress
	202	* until the entire list is destroyed. If this can't progress then there
	203	* is some bug related to object refcounting.
	204	*/
	205	while (!xa_empty(&ictx->objects)) {
	206	unsigned int destroyed = 0;
	207	unsigned long index;
	208
	209	xa_for_each(&ictx->objects, index, obj) {
	210	if (!refcount_dec_if_one(&obj->users))
	211	continue;
	212	destroyed++;
	213	xa_erase(&ictx->objects, index);
	214	iommufd_object_ops[obj->type].destroy(obj);
	215	kfree(obj);
	216	}
	217	/* Bug related to users refcount */
	218	if (WARN_ON(!destroyed))
	219	break;
	220	}
	221	kfree(ictx);
	222	return 0;
	223	}
	224
	225	static int iommufd_option(struct iommufd_ucmd *ucmd)
	226	{
	227	struct iommu_option *cmd = ucmd->cmd;
	228	int rc;
	229
	230	if (cmd->__reserved)
	231	return -EOPNOTSUPP;
	232
	233	switch (cmd->option_id) {
	234	case IOMMU_OPTION_RLIMIT_MODE:
	235	rc = iommufd_option_rlimit_mode(cmd, ucmd->ictx);
	236	break;
	237	case IOMMU_OPTION_HUGE_PAGES:
	238	rc = iommufd_ioas_option(ucmd);
	239	break;
	240	default:
	241	return -EOPNOTSUPP;
	242	}
	243	if (rc)
	244	return rc;
	245	if (copy_to_user(&((struct iommu_option __user *)ucmd->ubuffer)->val64,
	246	&cmd->val64, sizeof(cmd->val64)))
	247	return -EFAULT;
	248	return 0;
	249	}
	250
	251	union ucmd_buffer {
	252	struct iommu_destroy destroy;
	253	struct iommu_ioas_alloc alloc;
	254	struct iommu_ioas_allow_iovas allow_iovas;
	255	struct iommu_ioas_iova_ranges iova_ranges;
	256	struct iommu_ioas_map map;
	257	struct iommu_ioas_unmap unmap;
	258	#ifdef CONFIG_IOMMUFD_TEST
	259	struct iommu_test_cmd test;
	260	#endif
	261	};
	262
	263	struct iommufd_ioctl_op {
	264	unsigned int size;
	265	unsigned int min_size;
	266	unsigned int ioctl_num;
	267	int (execute)(struct iommufd_ucmd ucmd);
	268	};
	269
	270	#define IOCTL_OP(_ioctl, _fn, _struct, _last) \
	271	[_IOC_NR(_ioctl) - IOMMUFD_CMD_BASE] = { \
	272	.size = sizeof(_struct) + \
	273	BUILD_BUG_ON_ZERO(sizeof(union ucmd_buffer) < \
	274	sizeof(_struct)), \
	275	.min_size = offsetofend(_struct, _last), \
	276	.ioctl_num = _ioctl, \
	277	.execute = _fn, \
	278	}
	279	static const struct iommufd_ioctl_op iommufd_ioctl_ops[] = {
	280	IOCTL_OP(IOMMU_DESTROY, iommufd_destroy, struct iommu_destroy, id),
	281	IOCTL_OP(IOMMU_IOAS_ALLOC, iommufd_ioas_alloc_ioctl,
	282	struct iommu_ioas_alloc, out_ioas_id),
	283	IOCTL_OP(IOMMU_IOAS_ALLOW_IOVAS, iommufd_ioas_allow_iovas,
	284	struct iommu_ioas_allow_iovas, allowed_iovas),
	285	IOCTL_OP(IOMMU_IOAS_COPY, iommufd_ioas_copy, struct iommu_ioas_copy,
	286	src_iova),
	287	IOCTL_OP(IOMMU_IOAS_IOVA_RANGES, iommufd_ioas_iova_ranges,
	288	struct iommu_ioas_iova_ranges, out_iova_alignment),
	289	IOCTL_OP(IOMMU_IOAS_MAP, iommufd_ioas_map, struct iommu_ioas_map,
	290	iova),
	291	IOCTL_OP(IOMMU_IOAS_UNMAP, iommufd_ioas_unmap, struct iommu_ioas_unmap,
	292	length),
	293	IOCTL_OP(IOMMU_OPTION, iommufd_option, struct iommu_option,
	294	val64),
	295	IOCTL_OP(IOMMU_VFIO_IOAS, iommufd_vfio_ioas, struct iommu_vfio_ioas,
	296	__reserved),
	297	#ifdef CONFIG_IOMMUFD_TEST
	298	IOCTL_OP(IOMMU_TEST_CMD, iommufd_test, struct iommu_test_cmd, last),
	299	#endif
	300	};
	301
	302	static long iommufd_fops_ioctl(struct file *filp, unsigned int cmd,
	303	unsigned long arg)
	304	{
	305	struct iommufd_ctx *ictx = filp->private_data;
	306	const struct iommufd_ioctl_op *op;
	307	struct iommufd_ucmd ucmd = {};
	308	union ucmd_buffer buf;
	309	unsigned int nr;
	310	int ret;
	311
	312	nr = _IOC_NR(cmd);
	313	if (nr < IOMMUFD_CMD_BASE \|\|
	314	(nr - IOMMUFD_CMD_BASE) >= ARRAY_SIZE(iommufd_ioctl_ops))
	315	return iommufd_vfio_ioctl(ictx, cmd, arg);
	316
	317	ucmd.ictx = ictx;
	318	ucmd.ubuffer = (void __user *)arg;
	319	ret = get_user(ucmd.user_size, (u32 __user *)ucmd.ubuffer);
	320	if (ret)
	321	return ret;
	322
	323	op = &iommufd_ioctl_ops[nr - IOMMUFD_CMD_BASE];
	324	if (op->ioctl_num != cmd)
	325	return -ENOIOCTLCMD;
	326	if (ucmd.user_size < op->min_size)
	327	return -EINVAL;
	328
	329	ucmd.cmd = &buf;
	330	ret = copy_struct_from_user(ucmd.cmd, op->size, ucmd.ubuffer,
	331	ucmd.user_size);
	332	if (ret)
	333	return ret;
	334	ret = op->execute(&ucmd);
	335	return ret;
	336	}
	337
	338	static const struct file_operations iommufd_fops = {
	339	.owner = THIS_MODULE,
	340	.open = iommufd_fops_open,
	341	.release = iommufd_fops_release,
	342	.unlocked_ioctl = iommufd_fops_ioctl,
	343	};
	344
	345	/**
	346	* iommufd_ctx_get - Get a context reference
	347	* @ictx: Context to get
	348	*
	349	* The caller must already hold a valid reference to ictx.
	350	*/
	351	void iommufd_ctx_get(struct iommufd_ctx *ictx)
	352	{
	353	get_file(ictx->file);
	354	}
	355	EXPORT_SYMBOL_NS_GPL(iommufd_ctx_get, IOMMUFD);
	356
	357	/**
	358	* iommufd_ctx_from_file - Acquires a reference to the iommufd context
	359	* @file: File to obtain the reference from
	360	*
	361	* Returns a pointer to the iommufd_ctx, otherwise ERR_PTR. The struct file
	362	* remains owned by the caller and the caller must still do fput. On success
	363	* the caller is responsible to call iommufd_ctx_put().
	364	*/
	365	struct iommufd_ctx iommufd_ctx_from_file(struct file file)
	366	{
	367	struct iommufd_ctx *ictx;
	368
	369	if (file->f_op != &iommufd_fops)
	370	return ERR_PTR(-EBADFD);
	371	ictx = file->private_data;
	372	iommufd_ctx_get(ictx);
	373	return ictx;
	374	}
	375	EXPORT_SYMBOL_NS_GPL(iommufd_ctx_from_file, IOMMUFD);
	376
	377	/**
	378	* iommufd_ctx_put - Put back a reference
	379	* @ictx: Context to put back
	380	*/
	381	void iommufd_ctx_put(struct iommufd_ctx *ictx)
	382	{
	383	fput(ictx->file);
	384	}
	385	EXPORT_SYMBOL_NS_GPL(iommufd_ctx_put, IOMMUFD);
	386
	387	static const struct iommufd_object_ops iommufd_object_ops[] = {
	388	[IOMMUFD_OBJ_ACCESS] = {
	389	.destroy = iommufd_access_destroy_object,
	390	},
	391	[IOMMUFD_OBJ_DEVICE] = {
	392	.destroy = iommufd_device_destroy,
	393	},
	394	[IOMMUFD_OBJ_IOAS] = {
	395	.destroy = iommufd_ioas_destroy,
	396	},
	397	[IOMMUFD_OBJ_HW_PAGETABLE] = {
	398	.destroy = iommufd_hw_pagetable_destroy,
	399	},
	400	#ifdef CONFIG_IOMMUFD_TEST
	401	[IOMMUFD_OBJ_SELFTEST] = {
	402	.destroy = iommufd_selftest_destroy,
	403	},
	404	#endif
	405	};
	406
	407	static struct miscdevice iommu_misc_dev = {
	408	.minor = MISC_DYNAMIC_MINOR,
	409	.name = "iommu",
	410	.fops = &iommufd_fops,
	411	.nodename = "iommu",
	412	.mode = 0660,
	413	};
	414
	415
	416	static struct miscdevice vfio_misc_dev = {
	417	.minor = VFIO_MINOR,
	418	.name = "vfio",
	419	.fops = &iommufd_fops,
	420	.nodename = "vfio/vfio",
	421	.mode = 0666,
	422	};
	423
	424	static int __init iommufd_init(void)
	425	{
	426	int ret;
	427
	428	ret = misc_register(&iommu_misc_dev);
	429	if (ret)
	430	return ret;
	431
	432	if (IS_ENABLED(CONFIG_IOMMUFD_VFIO_CONTAINER)) {
	433	ret = misc_register(&vfio_misc_dev);
	434	if (ret)
	435	goto err_misc;
	436	}
	437	iommufd_test_init();
	438	return 0;
	439	err_misc:
	440	misc_deregister(&iommu_misc_dev);
	441	return ret;
	442	}
	443
	444	static void __exit iommufd_exit(void)
	445	{
	446	iommufd_test_exit();
	447	if (IS_ENABLED(CONFIG_IOMMUFD_VFIO_CONTAINER))
	448	misc_deregister(&vfio_misc_dev);
	449	misc_deregister(&iommu_misc_dev);
	450	}
	451
	452	module_init(iommufd_init);
	453	module_exit(iommufd_exit);
	454
	455	#if IS_ENABLED(CONFIG_IOMMUFD_VFIO_CONTAINER)
	456	MODULE_ALIAS_MISCDEV(VFIO_MINOR);
	457	MODULE_ALIAS("devname:vfio/vfio");
	458	#endif
	459	MODULE_DESCRIPTION("I/O Address Space Management for passthrough devices");
	460	MODULE_LICENSE("GPL");

drivers/iommu/iommufd/pages.c+1977−0

@@ -0,0 +1,1977 @@
	1	// SPDX-License-Identifier: GPL-2.0
	2	/* Copyright (c) 2021-2022, NVIDIA CORPORATION & AFFILIATES.
	3	*
	4	* The iopt_pages is the center of the storage and motion of PFNs. Each
	5	* iopt_pages represents a logical linear array of full PFNs. The array is 0
	6	* based and has npages in it. Accessors use 'index' to refer to the entry in
	7	* this logical array, regardless of its storage location.
	8	*
	9	* PFNs are stored in a tiered scheme:
	10	* 1) iopt_pages::pinned_pfns xarray
	11	* 2) An iommu_domain
	12	* 3) The origin of the PFNs, i.e. the userspace pointer
	13	*
	14	* PFN have to be copied between all combinations of tiers, depending on the
	15	* configuration.
	16	*
	17	* When a PFN is taken out of the userspace pointer it is pinned exactly once.
	18	* The storage locations of the PFN's index are tracked in the two interval
	19	* trees. If no interval includes the index then it is not pinned.
	20	*
	21	* If access_itree includes the PFN's index then an in-kernel access has
	22	* requested the page. The PFN is stored in the xarray so other requestors can
	23	* continue to find it.
	24	*
	25	* If the domains_itree includes the PFN's index then an iommu_domain is storing
	26	* the PFN and it can be read back using iommu_iova_to_phys(). To avoid
	27	* duplicating storage the xarray is not used if only iommu_domains are using
	28	* the PFN's index.
	29	*
	30	* As a general principle this is designed so that destroy never fails. This
	31	* means removing an iommu_domain or releasing a in-kernel access will not fail
	32	* due to insufficient memory. In practice this means some cases have to hold
	33	* PFNs in the xarray even though they are also being stored in an iommu_domain.
	34	*
	35	* While the iopt_pages can use an iommu_domain as storage, it does not have an
	36	* IOVA itself. Instead the iopt_area represents a range of IOVA and uses the
	37	* iopt_pages as the PFN provider. Multiple iopt_areas can share the iopt_pages
	38	* and reference their own slice of the PFN array, with sub page granularity.
	39	*
	40	* In this file the term 'last' indicates an inclusive and closed interval, eg
	41	* [0,0] refers to a single PFN. 'end' means an open range, eg [0,0) refers to
	42	* no PFNs.
	43	*
	44	* Be cautious of overflow. An IOVA can go all the way up to U64_MAX, so
	45	* last_iova + 1 can overflow. An iopt_pages index will always be much less than
	46	* ULONG_MAX so last_index + 1 cannot overflow.
	47	*/
	48	#include <linux/overflow.h>
	49	#include <linux/slab.h>
	50	#include <linux/iommu.h>
	51	#include <linux/sched/mm.h>
	52	#include <linux/highmem.h>
	53	#include <linux/kthread.h>
	54	#include <linux/iommufd.h>
	55
	56	#include "io_pagetable.h"
	57	#include "double_span.h"
	58
	59	#ifndef CONFIG_IOMMUFD_TEST
	60	#define TEMP_MEMORY_LIMIT 65536
	61	#else
	62	#define TEMP_MEMORY_LIMIT iommufd_test_memory_limit
	63	#endif
	64	#define BATCH_BACKUP_SIZE 32
	65
	66	/*
	67	* More memory makes pin_user_pages() and the batching more efficient, but as
	68	* this is only a performance optimization don't try too hard to get it. A 64k
	69	* allocation can hold about 26M of 4k pages and 13G of 2M pages in an
	70	* pfn_batch. Various destroy paths cannot fail and provide a small amount of
	71	* stack memory as a backup contingency. If backup_len is given this cannot
	72	* fail.
	73	*/
	74	static void temp_kmalloc(size_t size, void *backup, size_t backup_len)
	75	{
	76	void *res;
	77
	78	if (WARN_ON(*size == 0))
	79	return NULL;
	80
	81	if (*size < backup_len)
	82	return backup;
	83
	84	if (!backup && iommufd_should_fail())
	85	return NULL;
	86
	87	size = min_t(size_t, size, TEMP_MEMORY_LIMIT);
	88	res = kmalloc(*size, GFP_KERNEL \| __GFP_NOWARN \| __GFP_NORETRY);
	89	if (res)
	90	return res;
	91	*size = PAGE_SIZE;
	92	if (backup_len) {
	93	res = kmalloc(*size, GFP_KERNEL \| __GFP_NOWARN \| __GFP_NORETRY);
	94	if (res)
	95	return res;
	96	*size = backup_len;
	97	return backup;
	98	}
	99	return kmalloc(*size, GFP_KERNEL);
	100	}
	101
	102	void interval_tree_double_span_iter_update(
	103	struct interval_tree_double_span_iter *iter)
	104	{
	105	unsigned long last_hole = ULONG_MAX;
	106	unsigned int i;
	107
	108	for (i = 0; i != ARRAY_SIZE(iter->spans); i++) {
	109	if (interval_tree_span_iter_done(&iter->spans[i])) {
	110	iter->is_used = -1;
	111	return;
	112	}
	113
	114	if (iter->spans[i].is_hole) {
	115	last_hole = min(last_hole, iter->spans[i].last_hole);
	116	continue;
	117	}
	118
	119	iter->is_used = i + 1;
	120	iter->start_used = iter->spans[i].start_used;
	121	iter->last_used = min(iter->spans[i].last_used, last_hole);
	122	return;
	123	}
	124
	125	iter->is_used = 0;
	126	iter->start_hole = iter->spans[0].start_hole;
	127	iter->last_hole =
	128	min(iter->spans[0].last_hole, iter->spans[1].last_hole);
	129	}
	130
	131	void interval_tree_double_span_iter_first(
	132	struct interval_tree_double_span_iter *iter,
	133	struct rb_root_cached itree1, struct rb_root_cached itree2,
	134	unsigned long first_index, unsigned long last_index)
	135	{
	136	unsigned int i;
	137
	138	iter->itrees[0] = itree1;
	139	iter->itrees[1] = itree2;
	140	for (i = 0; i != ARRAY_SIZE(iter->spans); i++)
	141	interval_tree_span_iter_first(&iter->spans[i], iter->itrees[i],
	142	first_index, last_index);
	143	interval_tree_double_span_iter_update(iter);
	144	}
	145
	146	void interval_tree_double_span_iter_next(
	147	struct interval_tree_double_span_iter *iter)
	148	{
	149	unsigned int i;
	150
	151	if (iter->is_used == -1 \|\|
	152	iter->last_hole == iter->spans[0].last_index) {
	153	iter->is_used = -1;
	154	return;
	155	}
	156
	157	for (i = 0; i != ARRAY_SIZE(iter->spans); i++)
	158	interval_tree_span_iter_advance(
	159	&iter->spans[i], iter->itrees[i], iter->last_hole + 1);
	160	interval_tree_double_span_iter_update(iter);
	161	}
	162
	163	static void iopt_pages_add_npinned(struct iopt_pages *pages, size_t npages)
	164	{
	165	int rc;
	166
	167	rc = check_add_overflow(pages->npinned, npages, &pages->npinned);
	168	if (IS_ENABLED(CONFIG_IOMMUFD_TEST))
	169	WARN_ON(rc \|\| pages->npinned > pages->npages);
	170	}
	171
	172	static void iopt_pages_sub_npinned(struct iopt_pages *pages, size_t npages)
	173	{
	174	int rc;
	175
	176	rc = check_sub_overflow(pages->npinned, npages, &pages->npinned);
	177	if (IS_ENABLED(CONFIG_IOMMUFD_TEST))
	178	WARN_ON(rc \|\| pages->npinned > pages->npages);
	179	}
	180
	181	static void iopt_pages_err_unpin(struct iopt_pages *pages,
	182	unsigned long start_index,
	183	unsigned long last_index,
	184	struct page **page_list)
	185	{
	186	unsigned long npages = last_index - start_index + 1;
	187
	188	unpin_user_pages(page_list, npages);
	189	iopt_pages_sub_npinned(pages, npages);
	190	}
	191
	192	/*
	193	* index is the number of PAGE_SIZE units from the start of the area's
	194	* iopt_pages. If the iova is sub page-size then the area has an iova that
	195	* covers a portion of the first and last pages in the range.
	196	*/
	197	static unsigned long iopt_area_index_to_iova(struct iopt_area *area,
	198	unsigned long index)
	199	{
	200	if (IS_ENABLED(CONFIG_IOMMUFD_TEST))
	201	WARN_ON(index < iopt_area_index(area) \|\|
	202	index > iopt_area_last_index(area));
	203	index -= iopt_area_index(area);
	204	if (index == 0)
	205	return iopt_area_iova(area);
	206	return iopt_area_iova(area) - area->page_offset + index * PAGE_SIZE;
	207	}
	208
	209	static unsigned long iopt_area_index_to_iova_last(struct iopt_area *area,
	210	unsigned long index)
	211	{
	212	if (IS_ENABLED(CONFIG_IOMMUFD_TEST))
	213	WARN_ON(index < iopt_area_index(area) \|\|
	214	index > iopt_area_last_index(area));
	215	if (index == iopt_area_last_index(area))
	216	return iopt_area_last_iova(area);
	217	return iopt_area_iova(area) - area->page_offset +
	218	(index - iopt_area_index(area) + 1) * PAGE_SIZE - 1;
	219	}
	220
	221	static void iommu_unmap_nofail(struct iommu_domain *domain, unsigned long iova,
	222	size_t size)
	223	{
	224	size_t ret;
	225
	226	ret = iommu_unmap(domain, iova, size);
	227	/*
	228	* It is a logic error in this code or a driver bug if the IOMMU unmaps
	229	* something other than exactly as requested. This implies that the
	230	* iommu driver may not fail unmap for reasons beyond bad agruments.
	231	* Particularly, the iommu driver may not do a memory allocation on the
	232	* unmap path.
	233	*/
	234	WARN_ON(ret != size);
	235	}
	236
	237	static void iopt_area_unmap_domain_range(struct iopt_area *area,
	238	struct iommu_domain *domain,
	239	unsigned long start_index,
	240	unsigned long last_index)
	241	{
	242	unsigned long start_iova = iopt_area_index_to_iova(area, start_index);
	243
	244	iommu_unmap_nofail(domain, start_iova,
	245	iopt_area_index_to_iova_last(area, last_index) -
	246	start_iova + 1);
	247	}
	248
	249	static struct iopt_area iopt_pages_find_domain_area(struct iopt_pages pages,
	250	unsigned long index)
	251	{
	252	struct interval_tree_node *node;
	253
	254	node = interval_tree_iter_first(&pages->domains_itree, index, index);
	255	if (!node)
	256	return NULL;
	257	return container_of(node, struct iopt_area, pages_node);
	258	}
	259
	260	/*
	261	* A simple datastructure to hold a vector of PFNs, optimized for contiguous
	262	* PFNs. This is used as a temporary holding memory for shuttling pfns from one
	263	* place to another. Generally everything is made more efficient if operations
	264	* work on the largest possible grouping of pfns. eg fewer lock/unlock cycles,
	265	* better cache locality, etc
	266	*/
	267	struct pfn_batch {
	268	unsigned long *pfns;
	269	u32 *npfns;
	270	unsigned int array_size;
	271	unsigned int end;
	272	unsigned int total_pfns;
	273	};
	274
	275	static void batch_clear(struct pfn_batch *batch)
	276	{
	277	batch->total_pfns = 0;
	278	batch->end = 0;
	279	batch->pfns[0] = 0;
	280	batch->npfns[0] = 0;
	281	}
	282
	283	/*
	284	* Carry means we carry a portion of the final hugepage over to the front of the
	285	* batch
	286	*/
	287	static void batch_clear_carry(struct pfn_batch *batch, unsigned int keep_pfns)
	288	{
	289	if (!keep_pfns)
	290	return batch_clear(batch);
	291
	292	if (IS_ENABLED(CONFIG_IOMMUFD_TEST))
	293	WARN_ON(!batch->end \|\|
	294	batch->npfns[batch->end - 1] < keep_pfns);
	295
	296	batch->total_pfns = keep_pfns;
	297	batch->npfns[0] = keep_pfns;
	298	batch->pfns[0] = batch->pfns[batch->end - 1] +
	299	(batch->npfns[batch->end - 1] - keep_pfns);
	300	batch->end = 0;
	301	}
	302
	303	static void batch_skip_carry(struct pfn_batch *batch, unsigned int skip_pfns)
	304	{
	305	if (!batch->total_pfns)
	306	return;
	307	if (IS_ENABLED(CONFIG_IOMMUFD_TEST))
	308	WARN_ON(batch->total_pfns != batch->npfns[0]);
	309	skip_pfns = min(batch->total_pfns, skip_pfns);
	310	batch->pfns[0] += skip_pfns;
	311	batch->npfns[0] -= skip_pfns;
	312	batch->total_pfns -= skip_pfns;
	313	}
	314
	315	static int __batch_init(struct pfn_batch batch, size_t max_pages, void backup,
	316	size_t backup_len)
	317	{
	318	const size_t elmsz = sizeof(batch->pfns) + sizeof(batch->npfns);
	319	size_t size = max_pages * elmsz;
	320
	321	batch->pfns = temp_kmalloc(&size, backup, backup_len);
	322	if (!batch->pfns)
	323	return -ENOMEM;
	324	if (IS_ENABLED(CONFIG_IOMMUFD_TEST) && WARN_ON(size < elmsz))
	325	return -EINVAL;
	326	batch->array_size = size / elmsz;
	327	batch->npfns = (u32 *)(batch->pfns + batch->array_size);
	328	batch_clear(batch);
	329	return 0;
	330	}
	331
	332	static int batch_init(struct pfn_batch *batch, size_t max_pages)
	333	{
	334	return __batch_init(batch, max_pages, NULL, 0);
	335	}
	336
	337	static void batch_init_backup(struct pfn_batch *batch, size_t max_pages,
	338	void *backup, size_t backup_len)
	339	{
	340	__batch_init(batch, max_pages, backup, backup_len);
	341	}
	342
	343	static void batch_destroy(struct pfn_batch batch, void backup)
	344	{
	345	if (batch->pfns != backup)
	346	kfree(batch->pfns);
	347	}
	348
	349	/* true if the pfn was added, false otherwise */
	350	static bool batch_add_pfn(struct pfn_batch *batch, unsigned long pfn)
	351	{
	352	const unsigned int MAX_NPFNS = type_max(typeof(*batch->npfns));
	353
	354	if (batch->end &&
	355	pfn == batch->pfns[batch->end - 1] + batch->npfns[batch->end - 1] &&
	356	batch->npfns[batch->end - 1] != MAX_NPFNS) {
	357	batch->npfns[batch->end - 1]++;
	358	batch->total_pfns++;
	359	return true;
	360	}
	361	if (batch->end == batch->array_size)
	362	return false;
	363	batch->total_pfns++;
	364	batch->pfns[batch->end] = pfn;
	365	batch->npfns[batch->end] = 1;
	366	batch->end++;
	367	return true;
	368	}
	369
	370	/*
	371	* Fill the batch with pfns from the domain. When the batch is full, or it
	372	* reaches last_index, the function will return. The caller should use
	373	* batch->total_pfns to determine the starting point for the next iteration.
	374	*/
	375	static void batch_from_domain(struct pfn_batch *batch,
	376	struct iommu_domain *domain,
	377	struct iopt_area *area, unsigned long start_index,
	378	unsigned long last_index)
	379	{
	380	unsigned int page_offset = 0;
	381	unsigned long iova;
	382	phys_addr_t phys;
	383
	384	iova = iopt_area_index_to_iova(area, start_index);
	385	if (start_index == iopt_area_index(area))
	386	page_offset = area->page_offset;
	387	while (start_index <= last_index) {
	388	/*
	389	* This is pretty slow, it would be nice to get the page size
	390	* back from the driver, or have the driver directly fill the
	391	* batch.
	392	*/
	393	phys = iommu_iova_to_phys(domain, iova) - page_offset;
	394	if (!batch_add_pfn(batch, PHYS_PFN(phys)))
	395	return;
	396	iova += PAGE_SIZE - page_offset;
	397	page_offset = 0;
	398	start_index++;
	399	}
	400	}
	401
	402	static struct page *raw_pages_from_domain(struct iommu_domain domain,
	403	struct iopt_area *area,
	404	unsigned long start_index,
	405	unsigned long last_index,
	406	struct page **out_pages)
	407	{
	408	unsigned int page_offset = 0;
	409	unsigned long iova;
	410	phys_addr_t phys;
	411
	412	iova = iopt_area_index_to_iova(area, start_index);
	413	if (start_index == iopt_area_index(area))
	414	page_offset = area->page_offset;
	415	while (start_index <= last_index) {
	416	phys = iommu_iova_to_phys(domain, iova) - page_offset;
	417	*(out_pages++) = pfn_to_page(PHYS_PFN(phys));
	418	iova += PAGE_SIZE - page_offset;
	419	page_offset = 0;
	420	start_index++;
	421	}
	422	return out_pages;
	423	}
	424
	425	/* Continues reading a domain until we reach a discontinuity in the pfns. */
	426	static void batch_from_domain_continue(struct pfn_batch *batch,
	427	struct iommu_domain *domain,
	428	struct iopt_area *area,
	429	unsigned long start_index,
	430	unsigned long last_index)
	431	{
	432	unsigned int array_size = batch->array_size;
	433
	434	batch->array_size = batch->end;
	435	batch_from_domain(batch, domain, area, start_index, last_index);
	436	batch->array_size = array_size;
	437	}
	438
	439	/*
	440	* This is part of the VFIO compatibility support for VFIO_TYPE1_IOMMU. That
	441	* mode permits splitting a mapped area up, and then one of the splits is
	442	* unmapped. Doing this normally would cause us to violate our invariant of
	443	* pairing map/unmap. Thus, to support old VFIO compatibility disable support
	444	* for batching consecutive PFNs. All PFNs mapped into the iommu are done in
	445	* PAGE_SIZE units, not larger or smaller.
	446	*/
	447	static int batch_iommu_map_small(struct iommu_domain *domain,
	448	unsigned long iova, phys_addr_t paddr,
	449	size_t size, int prot)
	450	{
	451	unsigned long start_iova = iova;
	452	int rc;
	453
	454	if (IS_ENABLED(CONFIG_IOMMUFD_TEST))
	455	WARN_ON(paddr % PAGE_SIZE \|\| iova % PAGE_SIZE \|\|
	456	size % PAGE_SIZE);
	457
	458	while (size) {
	459	rc = iommu_map(domain, iova, paddr, PAGE_SIZE, prot);
	460	if (rc)
	461	goto err_unmap;
	462	iova += PAGE_SIZE;
	463	paddr += PAGE_SIZE;
	464	size -= PAGE_SIZE;
	465	}
	466	return 0;
	467
	468	err_unmap:
	469	if (start_iova != iova)
	470	iommu_unmap_nofail(domain, start_iova, iova - start_iova);
	471	return rc;
	472	}
	473
	474	static int batch_to_domain(struct pfn_batch batch, struct iommu_domain domain,
	475	struct iopt_area *area, unsigned long start_index)
	476	{
	477	bool disable_large_pages = area->iopt->disable_large_pages;
	478	unsigned long last_iova = iopt_area_last_iova(area);
	479	unsigned int page_offset = 0;
	480	unsigned long start_iova;
	481	unsigned long next_iova;
	482	unsigned int cur = 0;
	483	unsigned long iova;
	484	int rc;
	485
	486	/* The first index might be a partial page */
	487	if (start_index == iopt_area_index(area))
	488	page_offset = area->page_offset;
	489	next_iova = iova = start_iova =
	490	iopt_area_index_to_iova(area, start_index);
	491	while (cur < batch->end) {
	492	next_iova = min(last_iova + 1,
	493	next_iova + batch->npfns[cur] * PAGE_SIZE -
	494	page_offset);
	495	if (disable_large_pages)
	496	rc = batch_iommu_map_small(
	497	domain, iova,
	498	PFN_PHYS(batch->pfns[cur]) + page_offset,
	499	next_iova - iova, area->iommu_prot);
	500	else
	501	rc = iommu_map(domain, iova,
	502	PFN_PHYS(batch->pfns[cur]) + page_offset,
	503	next_iova - iova, area->iommu_prot);
	504	if (rc)
	505	goto err_unmap;
	506	iova = next_iova;
	507	page_offset = 0;
	508	cur++;
	509	}
	510	return 0;
	511	err_unmap:
	512	if (start_iova != iova)
	513	iommu_unmap_nofail(domain, start_iova, iova - start_iova);
	514	return rc;
	515	}
	516
	517	static void batch_from_xarray(struct pfn_batch batch, struct xarray xa,
	518	unsigned long start_index,
	519	unsigned long last_index)
	520	{
	521	XA_STATE(xas, xa, start_index);
	522	void *entry;
	523
	524	rcu_read_lock();
	525	while (true) {
	526	entry = xas_next(&xas);
	527	if (xas_retry(&xas, entry))
	528	continue;
	529	WARN_ON(!xa_is_value(entry));
	530	if (!batch_add_pfn(batch, xa_to_value(entry)) \|\|
	531	start_index == last_index)
	532	break;
	533	start_index++;
	534	}
	535	rcu_read_unlock();
	536	}
	537
	538	static void batch_from_xarray_clear(struct pfn_batch batch, struct xarray xa,
	539	unsigned long start_index,
	540	unsigned long last_index)
	541	{
	542	XA_STATE(xas, xa, start_index);
	543	void *entry;
	544
	545	xas_lock(&xas);
	546	while (true) {
	547	entry = xas_next(&xas);
	548	if (xas_retry(&xas, entry))
	549	continue;
	550	WARN_ON(!xa_is_value(entry));
	551	if (!batch_add_pfn(batch, xa_to_value(entry)))
	552	break;
	553	xas_store(&xas, NULL);
	554	if (start_index == last_index)
	555	break;
	556	start_index++;
	557	}
	558	xas_unlock(&xas);
	559	}
	560
	561	static void clear_xarray(struct xarray *xa, unsigned long start_index,
	562	unsigned long last_index)
	563	{
	564	XA_STATE(xas, xa, start_index);
	565	void *entry;
	566
	567	xas_lock(&xas);
	568	xas_for_each(&xas, entry, last_index)
	569	xas_store(&xas, NULL);
	570	xas_unlock(&xas);
	571	}
	572
	573	static int pages_to_xarray(struct xarray *xa, unsigned long start_index,
	574	unsigned long last_index, struct page **pages)
	575	{
	576	struct page **end_pages = pages + (last_index - start_index) + 1;
	577	struct page **half_pages = pages + (end_pages - pages) / 2;
	578	XA_STATE(xas, xa, start_index);
	579
	580	do {
	581	void *old;
	582
	583	xas_lock(&xas);
	584	while (pages != end_pages) {
	585	/* xarray does not participate in fault injection */
	586	if (pages == half_pages && iommufd_should_fail()) {
	587	xas_set_err(&xas, -EINVAL);
	588	xas_unlock(&xas);
	589	/* aka xas_destroy() */
	590	xas_nomem(&xas, GFP_KERNEL);
	591	goto err_clear;
	592	}
	593
	594	old = xas_store(&xas, xa_mk_value(page_to_pfn(*pages)));
	595	if (xas_error(&xas))
	596	break;
	597	WARN_ON(old);
	598	pages++;
	599	xas_next(&xas);
	600	}
	601	xas_unlock(&xas);
	602	} while (xas_nomem(&xas, GFP_KERNEL));
	603
	604	err_clear:
	605	if (xas_error(&xas)) {
	606	if (xas.xa_index != start_index)
	607	clear_xarray(xa, start_index, xas.xa_index - 1);
	608	return xas_error(&xas);
	609	}
	610	return 0;
	611	}
	612
	613	static void batch_from_pages(struct pfn_batch batch, struct page *pages,
	614	size_t npages)
	615	{
	616	struct page **end = pages + npages;
	617
	618	for (; pages != end; pages++)
	619	if (!batch_add_pfn(batch, page_to_pfn(*pages)))
	620	break;
	621	}
	622
	623	static void batch_unpin(struct pfn_batch batch, struct iopt_pages pages,
	624	unsigned int first_page_off, size_t npages)
	625	{
	626	unsigned int cur = 0;
	627
	628	while (first_page_off) {
	629	if (batch->npfns[cur] > first_page_off)
	630	break;
	631	first_page_off -= batch->npfns[cur];
	632	cur++;
	633	}
	634
	635	while (npages) {
	636	size_t to_unpin = min_t(size_t, npages,
	637	batch->npfns[cur] - first_page_off);
	638
	639	unpin_user_page_range_dirty_lock(
	640	pfn_to_page(batch->pfns[cur] + first_page_off),
	641	to_unpin, pages->writable);
	642	iopt_pages_sub_npinned(pages, to_unpin);
	643	cur++;
	644	first_page_off = 0;
	645	npages -= to_unpin;
	646	}
	647	}
	648
	649	static void copy_data_page(struct page page, void data, unsigned long offset,
	650	size_t length, unsigned int flags)
	651	{
	652	void *mem;
	653
	654	mem = kmap_local_page(page);
	655	if (flags & IOMMUFD_ACCESS_RW_WRITE) {
	656	memcpy(mem + offset, data, length);
	657	set_page_dirty_lock(page);
	658	} else {
	659	memcpy(data, mem + offset, length);
	660	}
	661	kunmap_local(mem);
	662	}
	663
	664	static unsigned long batch_rw(struct pfn_batch batch, void data,
	665	unsigned long offset, unsigned long length,
	666	unsigned int flags)
	667	{
	668	unsigned long copied = 0;
	669	unsigned int npage = 0;
	670	unsigned int cur = 0;
	671
	672	while (cur < batch->end) {
	673	unsigned long bytes = min(length, PAGE_SIZE - offset);
	674
	675	copy_data_page(pfn_to_page(batch->pfns[cur] + npage), data,
	676	offset, bytes, flags);
	677	offset = 0;
	678	length -= bytes;
	679	data += bytes;
	680	copied += bytes;
	681	npage++;
	682	if (npage == batch->npfns[cur]) {
	683	npage = 0;
	684	cur++;
	685	}
	686	if (!length)
	687	break;
	688	}
	689	return copied;
	690	}
	691
	692	/* pfn_reader_user is just the pin_user_pages() path */
	693	struct pfn_reader_user {
	694	struct page **upages;
	695	size_t upages_len;
	696	unsigned long upages_start;
	697	unsigned long upages_end;
	698	unsigned int gup_flags;
	699	/*
	700	* 1 means mmget() and mmap_read_lock(), 0 means only mmget(), -1 is
	701	* neither
	702	*/
	703	int locked;
	704	};
	705
	706	static void pfn_reader_user_init(struct pfn_reader_user *user,
	707	struct iopt_pages *pages)
	708	{
	709	user->upages = NULL;
	710	user->upages_start = 0;
	711	user->upages_end = 0;
	712	user->locked = -1;
	713
	714	user->gup_flags = FOLL_LONGTERM;
	715	if (pages->writable)
	716	user->gup_flags \|= FOLL_WRITE;
	717	}
	718
	719	static void pfn_reader_user_destroy(struct pfn_reader_user *user,
	720	struct iopt_pages *pages)
	721	{
	722	if (user->locked != -1) {
	723	if (user->locked)
	724	mmap_read_unlock(pages->source_mm);
	725	if (pages->source_mm != current->mm)
	726	mmput(pages->source_mm);
	727	user->locked = -1;
	728	}
	729
	730	kfree(user->upages);
	731	user->upages = NULL;
	732	}
	733
	734	static int pfn_reader_user_pin(struct pfn_reader_user *user,
	735	struct iopt_pages *pages,
	736	unsigned long start_index,
	737	unsigned long last_index)
	738	{
	739	bool remote_mm = pages->source_mm != current->mm;
	740	unsigned long npages;
	741	uintptr_t uptr;
	742	long rc;
	743
	744	if (IS_ENABLED(CONFIG_IOMMUFD_TEST) &&
	745	WARN_ON(last_index < start_index))
	746	return -EINVAL;
	747
	748	if (!user->upages) {
	749	/* All undone in pfn_reader_destroy() */
	750	user->upages_len =
	751	(last_index - start_index + 1) * sizeof(*user->upages);
	752	user->upages = temp_kmalloc(&user->upages_len, NULL, 0);
	753	if (!user->upages)
	754	return -ENOMEM;
	755	}
	756
	757	if (user->locked == -1) {
	758	/*
	759	* The majority of usages will run the map task within the mm
	760	* providing the pages, so we can optimize into
	761	* get_user_pages_fast()
	762	*/
	763	if (remote_mm) {
	764	if (!mmget_not_zero(pages->source_mm))
	765	return -EFAULT;
	766	}
	767	user->locked = 0;
	768	}
	769
	770	npages = min_t(unsigned long, last_index - start_index + 1,
	771	user->upages_len / sizeof(*user->upages));
	772
	773
	774	if (iommufd_should_fail())
	775	return -EFAULT;
	776
	777	uptr = (uintptr_t)(pages->uptr + start_index * PAGE_SIZE);
	778	if (!remote_mm)
	779	rc = pin_user_pages_fast(uptr, npages, user->gup_flags,
	780	user->upages);
	781	else {
	782	if (!user->locked) {
	783	mmap_read_lock(pages->source_mm);
	784	user->locked = 1;
	785	}
	786	rc = pin_user_pages_remote(pages->source_mm, uptr, npages,
	787	user->gup_flags, user->upages, NULL,
	788	&user->locked);
	789	}
	790	if (rc <= 0) {
	791	if (WARN_ON(!rc))
	792	return -EFAULT;
	793	return rc;
	794	}
	795	iopt_pages_add_npinned(pages, rc);
	796	user->upages_start = start_index;
	797	user->upages_end = start_index + rc;
	798	return 0;
	799	}
	800
	801	/* This is the "modern" and faster accounting method used by io_uring */
	802	static int incr_user_locked_vm(struct iopt_pages *pages, unsigned long npages)
	803	{
	804	unsigned long lock_limit;
	805	unsigned long cur_pages;
	806	unsigned long new_pages;
	807
	808	lock_limit = task_rlimit(pages->source_task, RLIMIT_MEMLOCK) >>
	809	PAGE_SHIFT;
	810	do {
	811	cur_pages = atomic_long_read(&pages->source_user->locked_vm);
	812	new_pages = cur_pages + npages;
	813	if (new_pages > lock_limit)
	814	return -ENOMEM;
	815	} while (atomic_long_cmpxchg(&pages->source_user->locked_vm, cur_pages,
	816	new_pages) != cur_pages);
	817	return 0;
	818	}
	819
	820	static void decr_user_locked_vm(struct iopt_pages *pages, unsigned long npages)
	821	{
	822	if (WARN_ON(atomic_long_read(&pages->source_user->locked_vm) < npages))
	823	return;
	824	atomic_long_sub(npages, &pages->source_user->locked_vm);
	825	}
	826
	827	/* This is the accounting method used for compatibility with VFIO */
	828	static int update_mm_locked_vm(struct iopt_pages *pages, unsigned long npages,
	829	bool inc, struct pfn_reader_user *user)
	830	{
	831	bool do_put = false;
	832	int rc;
	833
	834	if (user && user->locked) {
	835	mmap_read_unlock(pages->source_mm);
	836	user->locked = 0;
	837	/* If we had the lock then we also have a get */
	838	} else if ((!user \|\| !user->upages) &&
	839	pages->source_mm != current->mm) {
	840	if (!mmget_not_zero(pages->source_mm))
	841	return -EINVAL;
	842	do_put = true;
	843	}
	844
	845	mmap_write_lock(pages->source_mm);
	846	rc = __account_locked_vm(pages->source_mm, npages, inc,
	847	pages->source_task, false);
	848	mmap_write_unlock(pages->source_mm);
	849
	850	if (do_put)
	851	mmput(pages->source_mm);
	852	return rc;
	853	}
	854
	855	static int do_update_pinned(struct iopt_pages *pages, unsigned long npages,
	856	bool inc, struct pfn_reader_user *user)
	857	{
	858	int rc = 0;
	859
	860	switch (pages->account_mode) {
	861	case IOPT_PAGES_ACCOUNT_NONE:
	862	break;
	863	case IOPT_PAGES_ACCOUNT_USER:
	864	if (inc)
	865	rc = incr_user_locked_vm(pages, npages);
	866	else
	867	decr_user_locked_vm(pages, npages);
	868	break;
	869	case IOPT_PAGES_ACCOUNT_MM:
	870	rc = update_mm_locked_vm(pages, npages, inc, user);
	871	break;
	872	}
	873	if (rc)
	874	return rc;
	875
	876	pages->last_npinned = pages->npinned;
	877	if (inc)
	878	atomic64_add(npages, &pages->source_mm->pinned_vm);
	879	else
	880	atomic64_sub(npages, &pages->source_mm->pinned_vm);
	881	return 0;
	882	}
	883
	884	static void update_unpinned(struct iopt_pages *pages)
	885	{
	886	if (WARN_ON(pages->npinned > pages->last_npinned))
	887	return;
	888	if (pages->npinned == pages->last_npinned)
	889	return;
	890	do_update_pinned(pages, pages->last_npinned - pages->npinned, false,
	891	NULL);
	892	}
	893
	894	/*
	895	* Changes in the number of pages pinned is done after the pages have been read
	896	* and processed. If the user lacked the limit then the error unwind will unpin
	897	* everything that was just pinned. This is because it is expensive to calculate
	898	* how many pages we have already pinned within a range to generate an accurate
	899	* prediction in advance of doing the work to actually pin them.
	900	*/
	901	static int pfn_reader_user_update_pinned(struct pfn_reader_user *user,
	902	struct iopt_pages *pages)
	903	{
	904	unsigned long npages;
	905	bool inc;
	906
	907	lockdep_assert_held(&pages->mutex);
	908
	909	if (pages->npinned == pages->last_npinned)
	910	return 0;
	911
	912	if (pages->npinned < pages->last_npinned) {
	913	npages = pages->last_npinned - pages->npinned;
	914	inc = false;
	915	} else {
	916	if (iommufd_should_fail())
	917	return -ENOMEM;
	918	npages = pages->npinned - pages->last_npinned;
	919	inc = true;
	920	}
	921	return do_update_pinned(pages, npages, inc, user);
	922	}
	923
	924	/*
	925	* PFNs are stored in three places, in order of preference:
	926	* - The iopt_pages xarray. This is only populated if there is a
	927	* iopt_pages_access
	928	* - The iommu_domain under an area
	929	* - The original PFN source, ie pages->source_mm
	930	*
	931	* This iterator reads the pfns optimizing to load according to the
	932	* above order.
	933	*/
	934	struct pfn_reader {
	935	struct iopt_pages *pages;
	936	struct interval_tree_double_span_iter span;
	937	struct pfn_batch batch;
	938	unsigned long batch_start_index;
	939	unsigned long batch_end_index;
	940	unsigned long last_index;
	941
	942	struct pfn_reader_user user;
	943	};
	944
	945	static int pfn_reader_update_pinned(struct pfn_reader *pfns)
	946	{
	947	return pfn_reader_user_update_pinned(&pfns->user, pfns->pages);
	948	}
	949
	950	/*
	951	* The batch can contain a mixture of pages that are still in use and pages that
	952	* need to be unpinned. Unpin only pages that are not held anywhere else.
	953	*/
	954	static void pfn_reader_unpin(struct pfn_reader *pfns)
	955	{
	956	unsigned long last = pfns->batch_end_index - 1;
	957	unsigned long start = pfns->batch_start_index;
	958	struct interval_tree_double_span_iter span;
	959	struct iopt_pages *pages = pfns->pages;
	960
	961	lockdep_assert_held(&pages->mutex);
	962
	963	interval_tree_for_each_double_span(&span, &pages->access_itree,
	964	&pages->domains_itree, start, last) {
	965	if (span.is_used)
	966	continue;
	967
	968	batch_unpin(&pfns->batch, pages, span.start_hole - start,
	969	span.last_hole - span.start_hole + 1);
	970	}
	971	}
	972
	973	/* Process a single span to load it from the proper storage */
	974	static int pfn_reader_fill_span(struct pfn_reader *pfns)
	975	{
	976	struct interval_tree_double_span_iter *span = &pfns->span;
	977	unsigned long start_index = pfns->batch_end_index;
	978	struct iopt_area *area;
	979	int rc;
	980
	981	if (IS_ENABLED(CONFIG_IOMMUFD_TEST) &&
	982	WARN_ON(span->last_used < start_index))
	983	return -EINVAL;
	984
	985	if (span->is_used == 1) {
	986	batch_from_xarray(&pfns->batch, &pfns->pages->pinned_pfns,
	987	start_index, span->last_used);
	988	return 0;
	989	}
	990
	991	if (span->is_used == 2) {
	992	/*
	993	* Pull as many pages from the first domain we find in the
	994	* target span. If it is too small then we will be called again
	995	* and we'll find another area.
	996	*/
	997	area = iopt_pages_find_domain_area(pfns->pages, start_index);
	998	if (WARN_ON(!area))
	999	return -EINVAL;
	1000
	1001	/* The storage_domain cannot change without the pages mutex */
	1002	batch_from_domain(
	1003	&pfns->batch, area->storage_domain, area, start_index,
	1004	min(iopt_area_last_index(area), span->last_used));
	1005	return 0;
	1006	}
	1007
	1008	if (start_index >= pfns->user.upages_end) {
	1009	rc = pfn_reader_user_pin(&pfns->user, pfns->pages, start_index,
	1010	span->last_hole);
	1011	if (rc)
	1012	return rc;
	1013	}
	1014
	1015	batch_from_pages(&pfns->batch,
	1016	pfns->user.upages +
	1017	(start_index - pfns->user.upages_start),
	1018	pfns->user.upages_end - start_index);
	1019	return 0;
	1020	}
	1021
	1022	static bool pfn_reader_done(struct pfn_reader *pfns)
	1023	{
	1024	return pfns->batch_start_index == pfns->last_index + 1;
	1025	}
	1026
	1027	static int pfn_reader_next(struct pfn_reader *pfns)
	1028	{
	1029	int rc;
	1030
	1031	batch_clear(&pfns->batch);
	1032	pfns->batch_start_index = pfns->batch_end_index;
	1033
	1034	while (pfns->batch_end_index != pfns->last_index + 1) {
	1035	unsigned int npfns = pfns->batch.total_pfns;
	1036
	1037	if (IS_ENABLED(CONFIG_IOMMUFD_TEST) &&
	1038	WARN_ON(interval_tree_double_span_iter_done(&pfns->span)))
	1039	return -EINVAL;
	1040
	1041	rc = pfn_reader_fill_span(pfns);
	1042	if (rc)
	1043	return rc;
	1044
	1045	if (WARN_ON(!pfns->batch.total_pfns))
	1046	return -EINVAL;
	1047
	1048	pfns->batch_end_index =
	1049	pfns->batch_start_index + pfns->batch.total_pfns;
	1050	if (pfns->batch_end_index == pfns->span.last_used + 1)
	1051	interval_tree_double_span_iter_next(&pfns->span);
	1052
	1053	/* Batch is full */
	1054	if (npfns == pfns->batch.total_pfns)
	1055	return 0;
	1056	}
	1057	return 0;
	1058	}
	1059
	1060	static int pfn_reader_init(struct pfn_reader pfns, struct iopt_pages pages,
	1061	unsigned long start_index, unsigned long last_index)
	1062	{
	1063	int rc;
	1064
	1065	lockdep_assert_held(&pages->mutex);
	1066
	1067	pfns->pages = pages;
	1068	pfns->batch_start_index = start_index;
	1069	pfns->batch_end_index = start_index;
	1070	pfns->last_index = last_index;
	1071	pfn_reader_user_init(&pfns->user, pages);
	1072	rc = batch_init(&pfns->batch, last_index - start_index + 1);
	1073	if (rc)
	1074	return rc;
	1075	interval_tree_double_span_iter_first(&pfns->span, &pages->access_itree,
	1076	&pages->domains_itree, start_index,
	1077	last_index);
	1078	return 0;
	1079	}
	1080
	1081	/*
	1082	* There are many assertions regarding the state of pages->npinned vs
	1083	* pages->last_pinned, for instance something like unmapping a domain must only
	1084	* decrement the npinned, and pfn_reader_destroy() must be called only after all
	1085	* the pins are updated. This is fine for success flows, but error flows
	1086	* sometimes need to release the pins held inside the pfn_reader before going on
	1087	* to complete unmapping and releasing pins held in domains.
	1088	*/
	1089	static void pfn_reader_release_pins(struct pfn_reader *pfns)
	1090	{
	1091	struct iopt_pages *pages = pfns->pages;
	1092
	1093	if (pfns->user.upages_end > pfns->batch_end_index) {
	1094	size_t npages = pfns->user.upages_end - pfns->batch_end_index;
	1095
	1096	/* Any pages not transferred to the batch are just unpinned */
	1097	unpin_user_pages(pfns->user.upages + (pfns->batch_end_index -
	1098	pfns->user.upages_start),
	1099	npages);
	1100	iopt_pages_sub_npinned(pages, npages);
	1101	pfns->user.upages_end = pfns->batch_end_index;
	1102	}
	1103	if (pfns->batch_start_index != pfns->batch_end_index) {
	1104	pfn_reader_unpin(pfns);
	1105	pfns->batch_start_index = pfns->batch_end_index;
	1106	}
	1107	}
	1108
	1109	static void pfn_reader_destroy(struct pfn_reader *pfns)
	1110	{
	1111	struct iopt_pages *pages = pfns->pages;
	1112
	1113	pfn_reader_release_pins(pfns);
	1114	pfn_reader_user_destroy(&pfns->user, pfns->pages);
	1115	batch_destroy(&pfns->batch, NULL);
	1116	WARN_ON(pages->last_npinned != pages->npinned);
	1117	}
	1118
	1119	static int pfn_reader_first(struct pfn_reader pfns, struct iopt_pages pages,
	1120	unsigned long start_index, unsigned long last_index)
	1121	{
	1122	int rc;
	1123
	1124	if (IS_ENABLED(CONFIG_IOMMUFD_TEST) &&
	1125	WARN_ON(last_index < start_index))
	1126	return -EINVAL;
	1127
	1128	rc = pfn_reader_init(pfns, pages, start_index, last_index);
	1129	if (rc)
	1130	return rc;
	1131	rc = pfn_reader_next(pfns);
	1132	if (rc) {
	1133	pfn_reader_destroy(pfns);
	1134	return rc;
	1135	}
	1136	return 0;
	1137	}
	1138
	1139	struct iopt_pages iopt_alloc_pages(void __user uptr, unsigned long length,
	1140	bool writable)
	1141	{
	1142	struct iopt_pages *pages;
	1143
	1144	/*
	1145	* The iommu API uses size_t as the length, and protect the DIV_ROUND_UP
	1146	* below from overflow
	1147	*/
	1148	if (length > SIZE_MAX - PAGE_SIZE \|\| length == 0)
	1149	return ERR_PTR(-EINVAL);
	1150
	1151	pages = kzalloc(sizeof(*pages), GFP_KERNEL_ACCOUNT);
	1152	if (!pages)
	1153	return ERR_PTR(-ENOMEM);
	1154
	1155	kref_init(&pages->kref);
	1156	xa_init_flags(&pages->pinned_pfns, XA_FLAGS_ACCOUNT);
	1157	mutex_init(&pages->mutex);
	1158	pages->source_mm = current->mm;
	1159	mmgrab(pages->source_mm);
	1160	pages->uptr = (void __user *)ALIGN_DOWN((uintptr_t)uptr, PAGE_SIZE);
	1161	pages->npages = DIV_ROUND_UP(length + (uptr - pages->uptr), PAGE_SIZE);
	1162	pages->access_itree = RB_ROOT_CACHED;
	1163	pages->domains_itree = RB_ROOT_CACHED;
	1164	pages->writable = writable;
	1165	if (capable(CAP_IPC_LOCK))
	1166	pages->account_mode = IOPT_PAGES_ACCOUNT_NONE;
	1167	else
	1168	pages->account_mode = IOPT_PAGES_ACCOUNT_USER;
	1169	pages->source_task = current->group_leader;
	1170	get_task_struct(current->group_leader);
	1171	pages->source_user = get_uid(current_user());
	1172	return pages;
	1173	}
	1174
	1175	void iopt_release_pages(struct kref *kref)
	1176	{
	1177	struct iopt_pages *pages = container_of(kref, struct iopt_pages, kref);
	1178
	1179	WARN_ON(!RB_EMPTY_ROOT(&pages->access_itree.rb_root));
	1180	WARN_ON(!RB_EMPTY_ROOT(&pages->domains_itree.rb_root));
	1181	WARN_ON(pages->npinned);
	1182	WARN_ON(!xa_empty(&pages->pinned_pfns));
	1183	mmdrop(pages->source_mm);
	1184	mutex_destroy(&pages->mutex);
	1185	put_task_struct(pages->source_task);
	1186	free_uid(pages->source_user);
	1187	kfree(pages);
	1188	}
	1189
	1190	static void
	1191	iopt_area_unpin_domain(struct pfn_batch batch, struct iopt_area area,
	1192	struct iopt_pages pages, struct iommu_domain domain,
	1193	unsigned long start_index, unsigned long last_index,
	1194	unsigned long *unmapped_end_index,
	1195	unsigned long real_last_index)
	1196	{
	1197	while (start_index <= last_index) {
	1198	unsigned long batch_last_index;
	1199
	1200	if (*unmapped_end_index <= last_index) {
	1201	unsigned long start =
	1202	max(start_index, *unmapped_end_index);
	1203
	1204	batch_from_domain(batch, domain, area, start,
	1205	last_index);
	1206	batch_last_index = start + batch->total_pfns - 1;
	1207	} else {
	1208	batch_last_index = last_index;
	1209	}
	1210
	1211	/*
	1212	* unmaps must always 'cut' at a place where the pfns are not
	1213	* contiguous to pair with the maps that always install
	1214	* contiguous pages. Thus, if we have to stop unpinning in the
	1215	* middle of the domains we need to keep reading pfns until we
	1216	* find a cut point to do the unmap. The pfns we read are
	1217	* carried over and either skipped or integrated into the next
	1218	* batch.
	1219	*/
	1220	if (batch_last_index == last_index &&
	1221	last_index != real_last_index)
	1222	batch_from_domain_continue(batch, domain, area,
	1223	last_index + 1,
	1224	real_last_index);
	1225
	1226	if (*unmapped_end_index <= batch_last_index) {
	1227	iopt_area_unmap_domain_range(
	1228	area, domain, *unmapped_end_index,
	1229	start_index + batch->total_pfns - 1);
	1230	*unmapped_end_index = start_index + batch->total_pfns;
	1231	}
	1232
	1233	/* unpin must follow unmap */
	1234	batch_unpin(batch, pages, 0,
	1235	batch_last_index - start_index + 1);
	1236	start_index = batch_last_index + 1;
	1237
	1238	batch_clear_carry(batch,
	1239	*unmapped_end_index - batch_last_index - 1);
	1240	}
	1241	}
	1242
	1243	static void __iopt_area_unfill_domain(struct iopt_area *area,
	1244	struct iopt_pages *pages,
	1245	struct iommu_domain *domain,
	1246	unsigned long last_index)
	1247	{
	1248	struct interval_tree_double_span_iter span;
	1249	unsigned long start_index = iopt_area_index(area);
	1250	unsigned long unmapped_end_index = start_index;
	1251	u64 backup[BATCH_BACKUP_SIZE];
	1252	struct pfn_batch batch;
	1253
	1254	lockdep_assert_held(&pages->mutex);
	1255
	1256	/*
	1257	* For security we must not unpin something that is still DMA mapped,
	1258	* so this must unmap any IOVA before we go ahead and unpin the pages.
	1259	* This creates a complexity where we need to skip over unpinning pages
	1260	* held in the xarray, but continue to unmap from the domain.
	1261	*
	1262	* The domain unmap cannot stop in the middle of a contiguous range of
	1263	* PFNs. To solve this problem the unpinning step will read ahead to the
	1264	* end of any contiguous span, unmap that whole span, and then only
	1265	* unpin the leading part that does not have any accesses. The residual
	1266	* PFNs that were unmapped but not unpinned are called a "carry" in the
	1267	* batch as they are moved to the front of the PFN list and continue on
	1268	* to the next iteration(s).
	1269	*/
	1270	batch_init_backup(&batch, last_index + 1, backup, sizeof(backup));
	1271	interval_tree_for_each_double_span(&span, &pages->domains_itree,
	1272	&pages->access_itree, start_index,
	1273	last_index) {
	1274	if (span.is_used) {
	1275	batch_skip_carry(&batch,
	1276	span.last_used - span.start_used + 1);
	1277	continue;
	1278	}
	1279	iopt_area_unpin_domain(&batch, area, pages, domain,
	1280	span.start_hole, span.last_hole,
	1281	&unmapped_end_index, last_index);
	1282	}
	1283	/*
	1284	* If the range ends in a access then we do the residual unmap without
	1285	* any unpins.
	1286	*/
	1287	if (unmapped_end_index != last_index + 1)
	1288	iopt_area_unmap_domain_range(area, domain, unmapped_end_index,
	1289	last_index);
	1290	WARN_ON(batch.total_pfns);
	1291	batch_destroy(&batch, backup);
	1292	update_unpinned(pages);
	1293	}
	1294
	1295	static void iopt_area_unfill_partial_domain(struct iopt_area *area,
	1296	struct iopt_pages *pages,
	1297	struct iommu_domain *domain,
	1298	unsigned long end_index)
	1299	{
	1300	if (end_index != iopt_area_index(area))
	1301	__iopt_area_unfill_domain(area, pages, domain, end_index - 1);
	1302	}
	1303
	1304	/**
	1305	* iopt_area_unmap_domain() - Unmap without unpinning PFNs in a domain
	1306	* @area: The IOVA range to unmap
	1307	* @domain: The domain to unmap
	1308	*
	1309	* The caller must know that unpinning is not required, usually because there
	1310	* are other domains in the iopt.
	1311	*/
	1312	void iopt_area_unmap_domain(struct iopt_area area, struct iommu_domain domain)
	1313	{
	1314	iommu_unmap_nofail(domain, iopt_area_iova(area),
	1315	iopt_area_length(area));
	1316	}
	1317
	1318	/**
	1319	* iopt_area_unfill_domain() - Unmap and unpin PFNs in a domain
	1320	* @area: IOVA area to use
	1321	* @pages: page supplier for the area (area->pages is NULL)
	1322	* @domain: Domain to unmap from
	1323	*
	1324	* The domain should be removed from the domains_itree before calling. The
	1325	* domain will always be unmapped, but the PFNs may not be unpinned if there are
	1326	* still accesses.
	1327	*/
	1328	void iopt_area_unfill_domain(struct iopt_area area, struct iopt_pages pages,
	1329	struct iommu_domain *domain)
	1330	{
	1331	__iopt_area_unfill_domain(area, pages, domain,
	1332	iopt_area_last_index(area));
	1333	}
	1334
	1335	/**
	1336	* iopt_area_fill_domain() - Map PFNs from the area into a domain
	1337	* @area: IOVA area to use
	1338	* @domain: Domain to load PFNs into
	1339	*
	1340	* Read the pfns from the area's underlying iopt_pages and map them into the
	1341	* given domain. Called when attaching a new domain to an io_pagetable.
	1342	*/
	1343	int iopt_area_fill_domain(struct iopt_area area, struct iommu_domain domain)
	1344	{
	1345	unsigned long done_end_index;
	1346	struct pfn_reader pfns;
	1347	int rc;
	1348
	1349	lockdep_assert_held(&area->pages->mutex);
	1350
	1351	rc = pfn_reader_first(&pfns, area->pages, iopt_area_index(area),
	1352	iopt_area_last_index(area));
	1353	if (rc)
	1354	return rc;
	1355
	1356	while (!pfn_reader_done(&pfns)) {
	1357	done_end_index = pfns.batch_start_index;
	1358	rc = batch_to_domain(&pfns.batch, domain, area,
	1359	pfns.batch_start_index);
	1360	if (rc)
	1361	goto out_unmap;
	1362	done_end_index = pfns.batch_end_index;
	1363
	1364	rc = pfn_reader_next(&pfns);
	1365	if (rc)
	1366	goto out_unmap;
	1367	}
	1368
	1369	rc = pfn_reader_update_pinned(&pfns);
	1370	if (rc)
	1371	goto out_unmap;
	1372	goto out_destroy;
	1373
	1374	out_unmap:
	1375	pfn_reader_release_pins(&pfns);
	1376	iopt_area_unfill_partial_domain(area, area->pages, domain,
	1377	done_end_index);
	1378	out_destroy:
	1379	pfn_reader_destroy(&pfns);
	1380	return rc;
	1381	}
	1382
	1383	/**
	1384	* iopt_area_fill_domains() - Install PFNs into the area's domains
	1385	* @area: The area to act on
	1386	* @pages: The pages associated with the area (area->pages is NULL)
	1387	*
	1388	* Called during area creation. The area is freshly created and not inserted in
	1389	* the domains_itree yet. PFNs are read and loaded into every domain held in the
	1390	* area's io_pagetable and the area is installed in the domains_itree.
	1391	*
	1392	* On failure all domains are left unchanged.
	1393	*/
	1394	int iopt_area_fill_domains(struct iopt_area area, struct iopt_pages pages)
	1395	{
	1396	unsigned long done_first_end_index;
	1397	unsigned long done_all_end_index;
	1398	struct iommu_domain *domain;
	1399	unsigned long unmap_index;
	1400	struct pfn_reader pfns;
	1401	unsigned long index;
	1402	int rc;
	1403
	1404	lockdep_assert_held(&area->iopt->domains_rwsem);
	1405
	1406	if (xa_empty(&area->iopt->domains))
	1407	return 0;
	1408
	1409	mutex_lock(&pages->mutex);
	1410	rc = pfn_reader_first(&pfns, pages, iopt_area_index(area),
	1411	iopt_area_last_index(area));
	1412	if (rc)
	1413	goto out_unlock;
	1414
	1415	while (!pfn_reader_done(&pfns)) {
	1416	done_first_end_index = pfns.batch_end_index;
	1417	done_all_end_index = pfns.batch_start_index;
	1418	xa_for_each(&area->iopt->domains, index, domain) {
	1419	rc = batch_to_domain(&pfns.batch, domain, area,
	1420	pfns.batch_start_index);
	1421	if (rc)
	1422	goto out_unmap;
	1423	}
	1424	done_all_end_index = done_first_end_index;
	1425
	1426	rc = pfn_reader_next(&pfns);
	1427	if (rc)
	1428	goto out_unmap;
	1429	}
	1430	rc = pfn_reader_update_pinned(&pfns);
	1431	if (rc)
	1432	goto out_unmap;
	1433
	1434	area->storage_domain = xa_load(&area->iopt->domains, 0);
	1435	interval_tree_insert(&area->pages_node, &pages->domains_itree);
	1436	goto out_destroy;
	1437
	1438	out_unmap:
	1439	pfn_reader_release_pins(&pfns);
	1440	xa_for_each(&area->iopt->domains, unmap_index, domain) {
	1441	unsigned long end_index;
	1442
	1443	if (unmap_index < index)
	1444	end_index = done_first_end_index;
	1445	else
	1446	end_index = done_all_end_index;
	1447
	1448	/*
	1449	* The area is not yet part of the domains_itree so we have to
	1450	* manage the unpinning specially. The last domain does the
	1451	* unpin, every other domain is just unmapped.
	1452	*/
	1453	if (unmap_index != area->iopt->next_domain_id - 1) {
	1454	if (end_index != iopt_area_index(area))
	1455	iopt_area_unmap_domain_range(
	1456	area, domain, iopt_area_index(area),
	1457	end_index - 1);
	1458	} else {
	1459	iopt_area_unfill_partial_domain(area, pages, domain,
	1460	end_index);
	1461	}
	1462	}
	1463	out_destroy:
	1464	pfn_reader_destroy(&pfns);
	1465	out_unlock:
	1466	mutex_unlock(&pages->mutex);
	1467	return rc;
	1468	}
	1469
	1470	/**
	1471	* iopt_area_unfill_domains() - unmap PFNs from the area's domains
	1472	* @area: The area to act on
	1473	* @pages: The pages associated with the area (area->pages is NULL)
	1474	*
	1475	* Called during area destruction. This unmaps the iova's covered by all the
	1476	* area's domains and releases the PFNs.
	1477	*/
	1478	void iopt_area_unfill_domains(struct iopt_area area, struct iopt_pages pages)
	1479	{
	1480	struct io_pagetable *iopt = area->iopt;
	1481	struct iommu_domain *domain;
	1482	unsigned long index;
	1483
	1484	lockdep_assert_held(&iopt->domains_rwsem);
	1485
	1486	mutex_lock(&pages->mutex);
	1487	if (!area->storage_domain)
	1488	goto out_unlock;
	1489
	1490	xa_for_each(&iopt->domains, index, domain)
	1491	if (domain != area->storage_domain)
	1492	iopt_area_unmap_domain_range(
	1493	area, domain, iopt_area_index(area),
	1494	iopt_area_last_index(area));
	1495
	1496	interval_tree_remove(&area->pages_node, &pages->domains_itree);
	1497	iopt_area_unfill_domain(area, pages, area->storage_domain);
	1498	area->storage_domain = NULL;
	1499	out_unlock:
	1500	mutex_unlock(&pages->mutex);
	1501	}
	1502
	1503	static void iopt_pages_unpin_xarray(struct pfn_batch *batch,
	1504	struct iopt_pages *pages,
	1505	unsigned long start_index,
	1506	unsigned long end_index)
	1507	{
	1508	while (start_index <= end_index) {
	1509	batch_from_xarray_clear(batch, &pages->pinned_pfns, start_index,
	1510	end_index);
	1511	batch_unpin(batch, pages, 0, batch->total_pfns);
	1512	start_index += batch->total_pfns;
	1513	batch_clear(batch);
	1514	}
	1515	}
	1516
	1517	/**
	1518	* iopt_pages_unfill_xarray() - Update the xarry after removing an access
	1519	* @pages: The pages to act on
	1520	* @start_index: Starting PFN index
	1521	* @last_index: Last PFN index
	1522	*
	1523	* Called when an iopt_pages_access is removed, removes pages from the itree.
	1524	* The access should already be removed from the access_itree.
	1525	*/
	1526	void iopt_pages_unfill_xarray(struct iopt_pages *pages,
	1527	unsigned long start_index,
	1528	unsigned long last_index)
	1529	{
	1530	struct interval_tree_double_span_iter span;
	1531	u64 backup[BATCH_BACKUP_SIZE];
	1532	struct pfn_batch batch;
	1533	bool batch_inited = false;
	1534
	1535	lockdep_assert_held(&pages->mutex);
	1536
	1537	interval_tree_for_each_double_span(&span, &pages->access_itree,
	1538	&pages->domains_itree, start_index,
	1539	last_index) {
	1540	if (!span.is_used) {
	1541	if (!batch_inited) {
	1542	batch_init_backup(&batch,
	1543	last_index - start_index + 1,
	1544	backup, sizeof(backup));
	1545	batch_inited = true;
	1546	}
	1547	iopt_pages_unpin_xarray(&batch, pages, span.start_hole,
	1548	span.last_hole);
	1549	} else if (span.is_used == 2) {
	1550	/* Covered by a domain */
	1551	clear_xarray(&pages->pinned_pfns, span.start_used,
	1552	span.last_used);
	1553	}
	1554	/* Otherwise covered by an existing access */
	1555	}
	1556	if (batch_inited)
	1557	batch_destroy(&batch, backup);
	1558	update_unpinned(pages);
	1559	}
	1560
	1561	/**
	1562	* iopt_pages_fill_from_xarray() - Fast path for reading PFNs
	1563	* @pages: The pages to act on
	1564	* @start_index: The first page index in the range
	1565	* @last_index: The last page index in the range
	1566	* @out_pages: The output array to return the pages
	1567	*
	1568	* This can be called if the caller is holding a refcount on an
	1569	* iopt_pages_access that is known to have already been filled. It quickly reads
	1570	* the pages directly from the xarray.
	1571	*
	1572	* This is part of the SW iommu interface to read pages for in-kernel use.
	1573	*/
	1574	void iopt_pages_fill_from_xarray(struct iopt_pages *pages,
	1575	unsigned long start_index,
	1576	unsigned long last_index,
	1577	struct page **out_pages)
	1578	{
	1579	XA_STATE(xas, &pages->pinned_pfns, start_index);
	1580	void *entry;
	1581
	1582	rcu_read_lock();
	1583	while (start_index <= last_index) {
	1584	entry = xas_next(&xas);
	1585	if (xas_retry(&xas, entry))
	1586	continue;
	1587	WARN_ON(!xa_is_value(entry));
	1588	*(out_pages++) = pfn_to_page(xa_to_value(entry));
	1589	start_index++;
	1590	}
	1591	rcu_read_unlock();
	1592	}
	1593
	1594	static int iopt_pages_fill_from_domain(struct iopt_pages *pages,
	1595	unsigned long start_index,
	1596	unsigned long last_index,
	1597	struct page **out_pages)
	1598	{
	1599	while (start_index != last_index + 1) {
	1600	unsigned long domain_last;
	1601	struct iopt_area *area;
	1602
	1603	area = iopt_pages_find_domain_area(pages, start_index);
	1604	if (WARN_ON(!area))
	1605	return -EINVAL;
	1606
	1607	domain_last = min(iopt_area_last_index(area), last_index);
	1608	out_pages = raw_pages_from_domain(area->storage_domain, area,
	1609	start_index, domain_last,
	1610	out_pages);
	1611	start_index = domain_last + 1;
	1612	}
	1613	return 0;
	1614	}
	1615
	1616	static int iopt_pages_fill_from_mm(struct iopt_pages *pages,
	1617	struct pfn_reader_user *user,
	1618	unsigned long start_index,
	1619	unsigned long last_index,
	1620	struct page **out_pages)
	1621	{
	1622	unsigned long cur_index = start_index;
	1623	int rc;
	1624
	1625	while (cur_index != last_index + 1) {
	1626	user->upages = out_pages + (cur_index - start_index);
	1627	rc = pfn_reader_user_pin(user, pages, cur_index, last_index);
	1628	if (rc)
	1629	goto out_unpin;
	1630	cur_index = user->upages_end;
	1631	}
	1632	return 0;
	1633
	1634	out_unpin:
	1635	if (start_index != cur_index)
	1636	iopt_pages_err_unpin(pages, start_index, cur_index - 1,
	1637	out_pages);
	1638	return rc;
	1639	}
	1640
	1641	/**
	1642	* iopt_pages_fill_xarray() - Read PFNs
	1643	* @pages: The pages to act on
	1644	* @start_index: The first page index in the range
	1645	* @last_index: The last page index in the range
	1646	* @out_pages: The output array to return the pages, may be NULL
	1647	*
	1648	* This populates the xarray and returns the pages in out_pages. As the slow
	1649	* path this is able to copy pages from other storage tiers into the xarray.
	1650	*
	1651	* On failure the xarray is left unchanged.
	1652	*
	1653	* This is part of the SW iommu interface to read pages for in-kernel use.
	1654	*/
	1655	int iopt_pages_fill_xarray(struct iopt_pages *pages, unsigned long start_index,
	1656	unsigned long last_index, struct page **out_pages)
	1657	{
	1658	struct interval_tree_double_span_iter span;
	1659	unsigned long xa_end = start_index;
	1660	struct pfn_reader_user user;
	1661	int rc;
	1662
	1663	lockdep_assert_held(&pages->mutex);
	1664
	1665	pfn_reader_user_init(&user, pages);
	1666	user.upages_len = (last_index - start_index + 1) * sizeof(*out_pages);
	1667	interval_tree_for_each_double_span(&span, &pages->access_itree,
	1668	&pages->domains_itree, start_index,
	1669	last_index) {
	1670	struct page **cur_pages;
	1671
	1672	if (span.is_used == 1) {
	1673	cur_pages = out_pages + (span.start_used - start_index);
	1674	iopt_pages_fill_from_xarray(pages, span.start_used,
	1675	span.last_used, cur_pages);
	1676	continue;
	1677	}
	1678
	1679	if (span.is_used == 2) {
	1680	cur_pages = out_pages + (span.start_used - start_index);
	1681	iopt_pages_fill_from_domain(pages, span.start_used,
	1682	span.last_used, cur_pages);
	1683	rc = pages_to_xarray(&pages->pinned_pfns,
	1684	span.start_used, span.last_used,
	1685	cur_pages);
	1686	if (rc)
	1687	goto out_clean_xa;
	1688	xa_end = span.last_used + 1;
	1689	continue;
	1690	}
	1691
	1692	/* hole */
	1693	cur_pages = out_pages + (span.start_hole - start_index);
	1694	rc = iopt_pages_fill_from_mm(pages, &user, span.start_hole,
	1695	span.last_hole, cur_pages);
	1696	if (rc)
	1697	goto out_clean_xa;
	1698	rc = pages_to_xarray(&pages->pinned_pfns, span.start_hole,
	1699	span.last_hole, cur_pages);
	1700	if (rc) {
	1701	iopt_pages_err_unpin(pages, span.start_hole,
	1702	span.last_hole, cur_pages);
	1703	goto out_clean_xa;
	1704	}
	1705	xa_end = span.last_hole + 1;
	1706	}
	1707	rc = pfn_reader_user_update_pinned(&user, pages);
	1708	if (rc)
	1709	goto out_clean_xa;
	1710	user.upages = NULL;
	1711	pfn_reader_user_destroy(&user, pages);
	1712	return 0;
	1713
	1714	out_clean_xa:
	1715	if (start_index != xa_end)
	1716	iopt_pages_unfill_xarray(pages, start_index, xa_end - 1);
	1717	user.upages = NULL;
	1718	pfn_reader_user_destroy(&user, pages);
	1719	return rc;
	1720	}
	1721
	1722	/*
	1723	* This uses the pfn_reader instead of taking a shortcut by using the mm. It can
	1724	* do every scenario and is fully consistent with what an iommu_domain would
	1725	* see.
	1726	*/
	1727	static int iopt_pages_rw_slow(struct iopt_pages *pages,
	1728	unsigned long start_index,
	1729	unsigned long last_index, unsigned long offset,
	1730	void *data, unsigned long length,
	1731	unsigned int flags)
	1732	{
	1733	struct pfn_reader pfns;
	1734	int rc;
	1735
	1736	mutex_lock(&pages->mutex);
	1737
	1738	rc = pfn_reader_first(&pfns, pages, start_index, last_index);
	1739	if (rc)
	1740	goto out_unlock;
	1741
	1742	while (!pfn_reader_done(&pfns)) {
	1743	unsigned long done;
	1744
	1745	done = batch_rw(&pfns.batch, data, offset, length, flags);
	1746	data += done;
	1747	length -= done;
	1748	offset = 0;
	1749	pfn_reader_unpin(&pfns);
	1750
	1751	rc = pfn_reader_next(&pfns);
	1752	if (rc)
	1753	goto out_destroy;
	1754	}
	1755	if (WARN_ON(length != 0))
	1756	rc = -EINVAL;
	1757	out_destroy:
	1758	pfn_reader_destroy(&pfns);
	1759	out_unlock:
	1760	mutex_unlock(&pages->mutex);
	1761	return rc;
	1762	}
	1763
	1764	/*
	1765	* A medium speed path that still allows DMA inconsistencies, but doesn't do any
	1766	* memory allocations or interval tree searches.
	1767	*/
	1768	static int iopt_pages_rw_page(struct iopt_pages *pages, unsigned long index,
	1769	unsigned long offset, void *data,
	1770	unsigned long length, unsigned int flags)
	1771	{
	1772	struct page *page = NULL;
	1773	int rc;
	1774
	1775	if (!mmget_not_zero(pages->source_mm))
	1776	return iopt_pages_rw_slow(pages, index, index, offset, data,
	1777	length, flags);
	1778
	1779	if (iommufd_should_fail()) {
	1780	rc = -EINVAL;
	1781	goto out_mmput;
	1782	}
	1783
	1784	mmap_read_lock(pages->source_mm);
	1785	rc = pin_user_pages_remote(
	1786	pages->source_mm, (uintptr_t)(pages->uptr + index * PAGE_SIZE),
	1787	1, (flags & IOMMUFD_ACCESS_RW_WRITE) ? FOLL_WRITE : 0, &page,
	1788	NULL, NULL);
	1789	mmap_read_unlock(pages->source_mm);
	1790	if (rc != 1) {
	1791	if (WARN_ON(rc >= 0))
	1792	rc = -EINVAL;
	1793	goto out_mmput;
	1794	}
	1795	copy_data_page(page, data, offset, length, flags);
	1796	unpin_user_page(page);
	1797	rc = 0;
	1798
	1799	out_mmput:
	1800	mmput(pages->source_mm);
	1801	return rc;
	1802	}
	1803
	1804	/**
	1805	* iopt_pages_rw_access - Copy to/from a linear slice of the pages
	1806	* @pages: pages to act on
	1807	* @start_byte: First byte of pages to copy to/from
	1808	* @data: Kernel buffer to get/put the data
	1809	* @length: Number of bytes to copy
	1810	* @flags: IOMMUFD_ACCESS_RW_* flags
	1811	*
	1812	* This will find each page in the range, kmap it and then memcpy to/from
	1813	* the given kernel buffer.
	1814	*/
	1815	int iopt_pages_rw_access(struct iopt_pages *pages, unsigned long start_byte,
	1816	void *data, unsigned long length, unsigned int flags)
	1817	{
	1818	unsigned long start_index = start_byte / PAGE_SIZE;
	1819	unsigned long last_index = (start_byte + length - 1) / PAGE_SIZE;
	1820	bool change_mm = current->mm != pages->source_mm;
	1821	int rc = 0;
	1822
	1823	if (IS_ENABLED(CONFIG_IOMMUFD_TEST) &&
	1824	(flags & __IOMMUFD_ACCESS_RW_SLOW_PATH))
	1825	change_mm = true;
	1826
	1827	if ((flags & IOMMUFD_ACCESS_RW_WRITE) && !pages->writable)
	1828	return -EPERM;
	1829
	1830	if (!(flags & IOMMUFD_ACCESS_RW_KTHREAD) && change_mm) {
	1831	if (start_index == last_index)
	1832	return iopt_pages_rw_page(pages, start_index,
	1833	start_byte % PAGE_SIZE, data,
	1834	length, flags);
	1835	return iopt_pages_rw_slow(pages, start_index, last_index,
	1836	start_byte % PAGE_SIZE, data, length,
	1837	flags);
	1838	}
	1839
	1840	/*
	1841	* Try to copy using copy_to_user(). We do this as a fast path and
	1842	* ignore any pinning inconsistencies, unlike a real DMA path.
	1843	*/
	1844	if (change_mm) {
	1845	if (!mmget_not_zero(pages->source_mm))
	1846	return iopt_pages_rw_slow(pages, start_index,
	1847	last_index,
	1848	start_byte % PAGE_SIZE, data,
	1849	length, flags);
	1850	kthread_use_mm(pages->source_mm);
	1851	}
	1852
	1853	if (flags & IOMMUFD_ACCESS_RW_WRITE) {
	1854	if (copy_to_user(pages->uptr + start_byte, data, length))
	1855	rc = -EFAULT;
	1856	} else {
	1857	if (copy_from_user(data, pages->uptr + start_byte, length))
	1858	rc = -EFAULT;
	1859	}
	1860
	1861	if (change_mm) {
	1862	kthread_unuse_mm(pages->source_mm);
	1863	mmput(pages->source_mm);
	1864	}
	1865
	1866	return rc;
	1867	}
	1868
	1869	static struct iopt_pages_access *
	1870	iopt_pages_get_exact_access(struct iopt_pages *pages, unsigned long index,
	1871	unsigned long last)
	1872	{
	1873	struct interval_tree_node *node;
	1874
	1875	lockdep_assert_held(&pages->mutex);
	1876
	1877	/* There can be overlapping ranges in this interval tree */
	1878	for (node = interval_tree_iter_first(&pages->access_itree, index, last);
	1879	node; node = interval_tree_iter_next(node, index, last))
	1880	if (node->start == index && node->last == last)
	1881	return container_of(node, struct iopt_pages_access,
	1882	node);
	1883	return NULL;
	1884	}
	1885
	1886	/**
	1887	* iopt_area_add_access() - Record an in-knerel access for PFNs
	1888	* @area: The source of PFNs
	1889	* @start_index: First page index
	1890	* @last_index: Inclusive last page index
	1891	* @out_pages: Output list of struct page's representing the PFNs
	1892	* @flags: IOMMUFD_ACCESS_RW_* flags
	1893	*
	1894	* Record that an in-kernel access will be accessing the pages, ensure they are
	1895	* pinned, and return the PFNs as a simple list of 'struct page *'.
	1896	*
	1897	* This should be undone through a matching call to iopt_area_remove_access()
	1898	*/
	1899	int iopt_area_add_access(struct iopt_area *area, unsigned long start_index,
	1900	unsigned long last_index, struct page **out_pages,
	1901	unsigned int flags)
	1902	{
	1903	struct iopt_pages *pages = area->pages;
	1904	struct iopt_pages_access *access;
	1905	int rc;
	1906
	1907	if ((flags & IOMMUFD_ACCESS_RW_WRITE) && !pages->writable)
	1908	return -EPERM;
	1909
	1910	mutex_lock(&pages->mutex);
	1911	access = iopt_pages_get_exact_access(pages, start_index, last_index);
	1912	if (access) {
	1913	area->num_accesses++;
	1914	access->users++;
	1915	iopt_pages_fill_from_xarray(pages, start_index, last_index,
	1916	out_pages);
	1917	mutex_unlock(&pages->mutex);
	1918	return 0;
	1919	}
	1920
	1921	access = kzalloc(sizeof(*access), GFP_KERNEL_ACCOUNT);
	1922	if (!access) {
	1923	rc = -ENOMEM;
	1924	goto err_unlock;
	1925	}
	1926
	1927	rc = iopt_pages_fill_xarray(pages, start_index, last_index, out_pages);
	1928	if (rc)
	1929	goto err_free;
	1930
	1931	access->node.start = start_index;
	1932	access->node.last = last_index;
	1933	access->users = 1;
	1934	area->num_accesses++;
	1935	interval_tree_insert(&access->node, &pages->access_itree);
	1936	mutex_unlock(&pages->mutex);
	1937	return 0;
	1938
	1939	err_free:
	1940	kfree(access);
	1941	err_unlock:
	1942	mutex_unlock(&pages->mutex);
	1943	return rc;
	1944	}
	1945
	1946	/**
	1947	* iopt_area_remove_access() - Release an in-kernel access for PFNs
	1948	* @area: The source of PFNs
	1949	* @start_index: First page index
	1950	* @last_index: Inclusive last page index
	1951	*
	1952	* Undo iopt_area_add_access() and unpin the pages if necessary. The caller
	1953	* must stop using the PFNs before calling this.
	1954	*/
	1955	void iopt_area_remove_access(struct iopt_area *area, unsigned long start_index,
	1956	unsigned long last_index)
	1957	{
	1958	struct iopt_pages *pages = area->pages;
	1959	struct iopt_pages_access *access;
	1960
	1961	mutex_lock(&pages->mutex);
	1962	access = iopt_pages_get_exact_access(pages, start_index, last_index);
	1963	if (WARN_ON(!access))
	1964	goto out_unlock;
	1965
	1966	WARN_ON(area->num_accesses == 0 \|\| access->users == 0);
	1967	area->num_accesses--;
	1968	access->users--;
	1969	if (access->users)
	1970	goto out_unlock;
	1971
	1972	interval_tree_remove(&access->node, &pages->access_itree);
	1973	iopt_pages_unfill_xarray(pages, start_index, last_index);
	1974	kfree(access);
	1975	out_unlock:
	1976	mutex_unlock(&pages->mutex);
	1977	}

drivers/iommu/iommufd/selftest.c+853−0

@@ -0,0 +1,853 @@
	1	// SPDX-License-Identifier: GPL-2.0
	2	/* Copyright (c) 2021-2022, NVIDIA CORPORATION & AFFILIATES.
	3	*
	4	* Kernel side components to support tools/testing/selftests/iommu
	5	*/
	6	#include <linux/slab.h>
	7	#include <linux/iommu.h>
	8	#include <linux/xarray.h>
	9	#include <linux/file.h>
	10	#include <linux/anon_inodes.h>
	11	#include <linux/fault-inject.h>
	12	#include <uapi/linux/iommufd.h>
	13
	14	#include "io_pagetable.h"
	15	#include "iommufd_private.h"
	16	#include "iommufd_test.h"
	17
	18	static DECLARE_FAULT_ATTR(fail_iommufd);
	19	static struct dentry *dbgfs_root;
	20
	21	size_t iommufd_test_memory_limit = 65536;
	22
	23	enum {
	24	MOCK_IO_PAGE_SIZE = PAGE_SIZE / 2,
	25
	26	/*
	27	* Like a real page table alignment requires the low bits of the address
	28	* to be zero. xarray also requires the high bit to be zero, so we store
	29	* the pfns shifted. The upper bits are used for metadata.
	30	*/
	31	MOCK_PFN_MASK = ULONG_MAX / MOCK_IO_PAGE_SIZE,
	32
	33	_MOCK_PFN_START = MOCK_PFN_MASK + 1,
	34	MOCK_PFN_START_IOVA = _MOCK_PFN_START,
	35	MOCK_PFN_LAST_IOVA = _MOCK_PFN_START,
	36	};
	37
	38	/*
	39	* Syzkaller has trouble randomizing the correct iova to use since it is linked
	40	* to the map ioctl's output, and it has no ide about that. So, simplify things.
	41	* In syzkaller mode the 64 bit IOVA is converted into an nth area and offset
	42	* value. This has a much smaller randomization space and syzkaller can hit it.
	43	*/
	44	static unsigned long iommufd_test_syz_conv_iova(struct io_pagetable *iopt,
	45	u64 *iova)
	46	{
	47	struct syz_layout {
	48	__u32 nth_area;
	49	__u32 offset;
	50	};
	51	struct syz_layout syz = (void )iova;
	52	unsigned int nth = syz->nth_area;
	53	struct iopt_area *area;
	54
	55	down_read(&iopt->iova_rwsem);
	56	for (area = iopt_area_iter_first(iopt, 0, ULONG_MAX); area;
	57	area = iopt_area_iter_next(area, 0, ULONG_MAX)) {
	58	if (nth == 0) {
	59	up_read(&iopt->iova_rwsem);
	60	return iopt_area_iova(area) + syz->offset;
	61	}
	62	nth--;
	63	}
	64	up_read(&iopt->iova_rwsem);
	65
	66	return 0;
	67	}
	68
	69	void iommufd_test_syz_conv_iova_id(struct iommufd_ucmd *ucmd,
	70	unsigned int ioas_id, u64 iova, u32 flags)
	71	{
	72	struct iommufd_ioas *ioas;
	73
	74	if (!(*flags & MOCK_FLAGS_ACCESS_SYZ))
	75	return;
	76	*flags &= ~(u32)MOCK_FLAGS_ACCESS_SYZ;
	77
	78	ioas = iommufd_get_ioas(ucmd, ioas_id);
	79	if (IS_ERR(ioas))
	80	return;
	81	*iova = iommufd_test_syz_conv_iova(&ioas->iopt, iova);
	82	iommufd_put_object(&ioas->obj);
	83	}
	84
	85	struct mock_iommu_domain {
	86	struct iommu_domain domain;
	87	struct xarray pfns;
	88	};
	89
	90	enum selftest_obj_type {
	91	TYPE_IDEV,
	92	};
	93
	94	struct selftest_obj {
	95	struct iommufd_object obj;
	96	enum selftest_obj_type type;
	97
	98	union {
	99	struct {
	100	struct iommufd_hw_pagetable *hwpt;
	101	struct iommufd_ctx *ictx;
	102	struct device mock_dev;
	103	} idev;
	104	};
	105	};
	106
	107	static struct iommu_domain *mock_domain_alloc(unsigned int iommu_domain_type)
	108	{
	109	struct mock_iommu_domain *mock;
	110
	111	if (WARN_ON(iommu_domain_type != IOMMU_DOMAIN_UNMANAGED))
	112	return NULL;
	113
	114	mock = kzalloc(sizeof(*mock), GFP_KERNEL);
	115	if (!mock)
	116	return NULL;
	117	mock->domain.geometry.aperture_start = MOCK_APERTURE_START;
	118	mock->domain.geometry.aperture_end = MOCK_APERTURE_LAST;
	119	mock->domain.pgsize_bitmap = MOCK_IO_PAGE_SIZE;
	120	xa_init(&mock->pfns);
	121	return &mock->domain;
	122	}
	123
	124	static void mock_domain_free(struct iommu_domain *domain)
	125	{
	126	struct mock_iommu_domain *mock =
	127	container_of(domain, struct mock_iommu_domain, domain);
	128
	129	WARN_ON(!xa_empty(&mock->pfns));
	130	kfree(mock);
	131	}
	132
	133	static int mock_domain_map_pages(struct iommu_domain *domain,
	134	unsigned long iova, phys_addr_t paddr,
	135	size_t pgsize, size_t pgcount, int prot,
	136	gfp_t gfp, size_t *mapped)
	137	{
	138	struct mock_iommu_domain *mock =
	139	container_of(domain, struct mock_iommu_domain, domain);
	140	unsigned long flags = MOCK_PFN_START_IOVA;
	141	unsigned long start_iova = iova;
	142
	143	/*
	144	* xarray does not reliably work with fault injection because it does a
	145	* retry allocation, so put our own failure point.
	146	*/
	147	if (iommufd_should_fail())
	148	return -ENOENT;
	149
	150	WARN_ON(iova % MOCK_IO_PAGE_SIZE);
	151	WARN_ON(pgsize % MOCK_IO_PAGE_SIZE);
	152	for (; pgcount; pgcount--) {
	153	size_t cur;
	154
	155	for (cur = 0; cur != pgsize; cur += MOCK_IO_PAGE_SIZE) {
	156	void *old;
	157
	158	if (pgcount == 1 && cur + MOCK_IO_PAGE_SIZE == pgsize)
	159	flags = MOCK_PFN_LAST_IOVA;
	160	old = xa_store(&mock->pfns, iova / MOCK_IO_PAGE_SIZE,
	161	xa_mk_value((paddr / MOCK_IO_PAGE_SIZE) \|
	162	flags),
	163	gfp);
	164	if (xa_is_err(old)) {
	165	for (; start_iova != iova;
	166	start_iova += MOCK_IO_PAGE_SIZE)
	167	xa_erase(&mock->pfns,
	168	start_iova /
	169	MOCK_IO_PAGE_SIZE);
	170	return xa_err(old);
	171	}
	172	WARN_ON(old);
	173	iova += MOCK_IO_PAGE_SIZE;
	174	paddr += MOCK_IO_PAGE_SIZE;
	175	*mapped += MOCK_IO_PAGE_SIZE;
	176	flags = 0;
	177	}
	178	}
	179	return 0;
	180	}
	181
	182	static size_t mock_domain_unmap_pages(struct iommu_domain *domain,
	183	unsigned long iova, size_t pgsize,
	184	size_t pgcount,
	185	struct iommu_iotlb_gather *iotlb_gather)
	186	{
	187	struct mock_iommu_domain *mock =
	188	container_of(domain, struct mock_iommu_domain, domain);
	189	bool first = true;
	190	size_t ret = 0;
	191	void *ent;
	192
	193	WARN_ON(iova % MOCK_IO_PAGE_SIZE);
	194	WARN_ON(pgsize % MOCK_IO_PAGE_SIZE);
	195
	196	for (; pgcount; pgcount--) {
	197	size_t cur;
	198
	199	for (cur = 0; cur != pgsize; cur += MOCK_IO_PAGE_SIZE) {
	200	ent = xa_erase(&mock->pfns, iova / MOCK_IO_PAGE_SIZE);
	201	WARN_ON(!ent);
	202	/*
	203	* iommufd generates unmaps that must be a strict
	204	* superset of the map's performend So every starting
	205	* IOVA should have been an iova passed to map, and the
	206	*
	207	* First IOVA must be present and have been a first IOVA
	208	* passed to map_pages
	209	*/
	210	if (first) {
	211	WARN_ON(!(xa_to_value(ent) &
	212	MOCK_PFN_START_IOVA));
	213	first = false;
	214	}
	215	if (pgcount == 1 && cur + MOCK_IO_PAGE_SIZE == pgsize)
	216	WARN_ON(!(xa_to_value(ent) &
	217	MOCK_PFN_LAST_IOVA));
	218
	219	iova += MOCK_IO_PAGE_SIZE;
	220	ret += MOCK_IO_PAGE_SIZE;
	221	}
	222	}
	223	return ret;
	224	}
	225
	226	static phys_addr_t mock_domain_iova_to_phys(struct iommu_domain *domain,
	227	dma_addr_t iova)
	228	{
	229	struct mock_iommu_domain *mock =
	230	container_of(domain, struct mock_iommu_domain, domain);
	231	void *ent;
	232
	233	WARN_ON(iova % MOCK_IO_PAGE_SIZE);
	234	ent = xa_load(&mock->pfns, iova / MOCK_IO_PAGE_SIZE);
	235	WARN_ON(!ent);
	236	return (xa_to_value(ent) & MOCK_PFN_MASK) * MOCK_IO_PAGE_SIZE;
	237	}
	238
	239	static const struct iommu_ops mock_ops = {
	240	.owner = THIS_MODULE,
	241	.pgsize_bitmap = MOCK_IO_PAGE_SIZE,
	242	.domain_alloc = mock_domain_alloc,
	243	.default_domain_ops =
	244	&(struct iommu_domain_ops){
	245	.free = mock_domain_free,
	246	.map_pages = mock_domain_map_pages,
	247	.unmap_pages = mock_domain_unmap_pages,
	248	.iova_to_phys = mock_domain_iova_to_phys,
	249	},
	250	};
	251
	252	static inline struct iommufd_hw_pagetable *
	253	get_md_pagetable(struct iommufd_ucmd *ucmd, u32 mockpt_id,
	254	struct mock_iommu_domain **mock)
	255	{
	256	struct iommufd_hw_pagetable *hwpt;
	257	struct iommufd_object *obj;
	258
	259	obj = iommufd_get_object(ucmd->ictx, mockpt_id,
	260	IOMMUFD_OBJ_HW_PAGETABLE);
	261	if (IS_ERR(obj))
	262	return ERR_CAST(obj);
	263	hwpt = container_of(obj, struct iommufd_hw_pagetable, obj);
	264	if (hwpt->domain->ops != mock_ops.default_domain_ops) {
	265	iommufd_put_object(&hwpt->obj);
	266	return ERR_PTR(-EINVAL);
	267	}
	268	*mock = container_of(hwpt->domain, struct mock_iommu_domain, domain);
	269	return hwpt;
	270	}
	271
	272	/* Create an hw_pagetable with the mock domain so we can test the domain ops */
	273	static int iommufd_test_mock_domain(struct iommufd_ucmd *ucmd,
	274	struct iommu_test_cmd *cmd)
	275	{
	276	static struct bus_type mock_bus = { .iommu_ops = &mock_ops };
	277	struct iommufd_hw_pagetable *hwpt;
	278	struct selftest_obj *sobj;
	279	struct iommufd_ioas *ioas;
	280	int rc;
	281
	282	ioas = iommufd_get_ioas(ucmd, cmd->id);
	283	if (IS_ERR(ioas))
	284	return PTR_ERR(ioas);
	285
	286	sobj = iommufd_object_alloc(ucmd->ictx, sobj, IOMMUFD_OBJ_SELFTEST);
	287	if (IS_ERR(sobj)) {
	288	rc = PTR_ERR(sobj);
	289	goto out_ioas;
	290	}
	291	sobj->idev.ictx = ucmd->ictx;
	292	sobj->type = TYPE_IDEV;
	293	sobj->idev.mock_dev.bus = &mock_bus;
	294
	295	hwpt = iommufd_device_selftest_attach(ucmd->ictx, ioas,
	296	&sobj->idev.mock_dev);
	297	if (IS_ERR(hwpt)) {
	298	rc = PTR_ERR(hwpt);
	299	goto out_sobj;
	300	}
	301	sobj->idev.hwpt = hwpt;
	302
	303	/* Userspace must destroy both of these IDs to destroy the object */
	304	cmd->mock_domain.out_hwpt_id = hwpt->obj.id;
	305	cmd->mock_domain.out_device_id = sobj->obj.id;
	306	iommufd_object_finalize(ucmd->ictx, &sobj->obj);
	307	iommufd_put_object(&ioas->obj);
	308	return iommufd_ucmd_respond(ucmd, sizeof(*cmd));
	309
	310	out_sobj:
	311	iommufd_object_abort(ucmd->ictx, &sobj->obj);
	312	out_ioas:
	313	iommufd_put_object(&ioas->obj);
	314	return rc;
	315	}
	316
	317	/* Add an additional reserved IOVA to the IOAS */
	318	static int iommufd_test_add_reserved(struct iommufd_ucmd *ucmd,
	319	unsigned int mockpt_id,
	320	unsigned long start, size_t length)
	321	{
	322	struct iommufd_ioas *ioas;
	323	int rc;
	324
	325	ioas = iommufd_get_ioas(ucmd, mockpt_id);
	326	if (IS_ERR(ioas))
	327	return PTR_ERR(ioas);
	328	down_write(&ioas->iopt.iova_rwsem);
	329	rc = iopt_reserve_iova(&ioas->iopt, start, start + length - 1, NULL);
	330	up_write(&ioas->iopt.iova_rwsem);
	331	iommufd_put_object(&ioas->obj);
	332	return rc;
	333	}
	334
	335	/* Check that every pfn under each iova matches the pfn under a user VA */
	336	static int iommufd_test_md_check_pa(struct iommufd_ucmd *ucmd,
	337	unsigned int mockpt_id, unsigned long iova,
	338	size_t length, void __user *uptr)
	339	{
	340	struct iommufd_hw_pagetable *hwpt;
	341	struct mock_iommu_domain *mock;
	342	int rc;
	343
	344	if (iova % MOCK_IO_PAGE_SIZE \|\| length % MOCK_IO_PAGE_SIZE \|\|
	345	(uintptr_t)uptr % MOCK_IO_PAGE_SIZE)
	346	return -EINVAL;
	347
	348	hwpt = get_md_pagetable(ucmd, mockpt_id, &mock);
	349	if (IS_ERR(hwpt))
	350	return PTR_ERR(hwpt);
	351
	352	for (; length; length -= MOCK_IO_PAGE_SIZE) {
	353	struct page *pages[1];
	354	unsigned long pfn;
	355	long npages;
	356	void *ent;
	357
	358	npages = get_user_pages_fast((uintptr_t)uptr & PAGE_MASK, 1, 0,
	359	pages);
	360	if (npages < 0) {
	361	rc = npages;
	362	goto out_put;
	363	}
	364	if (WARN_ON(npages != 1)) {
	365	rc = -EFAULT;
	366	goto out_put;
	367	}
	368	pfn = page_to_pfn(pages[0]);
	369	put_page(pages[0]);
	370
	371	ent = xa_load(&mock->pfns, iova / MOCK_IO_PAGE_SIZE);
	372	if (!ent \|\|
	373	(xa_to_value(ent) & MOCK_PFN_MASK) * MOCK_IO_PAGE_SIZE !=
	374	pfn * PAGE_SIZE + ((uintptr_t)uptr % PAGE_SIZE)) {
	375	rc = -EINVAL;
	376	goto out_put;
	377	}
	378	iova += MOCK_IO_PAGE_SIZE;
	379	uptr += MOCK_IO_PAGE_SIZE;
	380	}
	381	rc = 0;
	382
	383	out_put:
	384	iommufd_put_object(&hwpt->obj);
	385	return rc;
	386	}
	387
	388	/* Check that the page ref count matches, to look for missing pin/unpins */
	389	static int iommufd_test_md_check_refs(struct iommufd_ucmd *ucmd,
	390	void __user *uptr, size_t length,
	391	unsigned int refs)
	392	{
	393	if (length % PAGE_SIZE \|\| (uintptr_t)uptr % PAGE_SIZE)
	394	return -EINVAL;
	395
	396	for (; length; length -= PAGE_SIZE) {
	397	struct page *pages[1];
	398	long npages;
	399
	400	npages = get_user_pages_fast((uintptr_t)uptr, 1, 0, pages);
	401	if (npages < 0)
	402	return npages;
	403	if (WARN_ON(npages != 1))
	404	return -EFAULT;
	405	if (!PageCompound(pages[0])) {
	406	unsigned int count;
	407
	408	count = page_ref_count(pages[0]);
	409	if (count / GUP_PIN_COUNTING_BIAS != refs) {
	410	put_page(pages[0]);
	411	return -EIO;
	412	}
	413	}
	414	put_page(pages[0]);
	415	uptr += PAGE_SIZE;
	416	}
	417	return 0;
	418	}
	419
	420	struct selftest_access {
	421	struct iommufd_access *access;
	422	struct file *file;
	423	struct mutex lock;
	424	struct list_head items;
	425	unsigned int next_id;
	426	bool destroying;
	427	};
	428
	429	struct selftest_access_item {
	430	struct list_head items_elm;
	431	unsigned long iova;
	432	size_t length;
	433	unsigned int id;
	434	};
	435
	436	static const struct file_operations iommfd_test_staccess_fops;
	437
	438	static struct selftest_access *iommufd_access_get(int fd)
	439	{
	440	struct file *file;
	441
	442	file = fget(fd);
	443	if (!file)
	444	return ERR_PTR(-EBADFD);
	445
	446	if (file->f_op != &iommfd_test_staccess_fops) {
	447	fput(file);
	448	return ERR_PTR(-EBADFD);
	449	}
	450	return file->private_data;
	451	}
	452
	453	static void iommufd_test_access_unmap(void *data, unsigned long iova,
	454	unsigned long length)
	455	{
	456	unsigned long iova_last = iova + length - 1;
	457	struct selftest_access *staccess = data;
	458	struct selftest_access_item *item;
	459	struct selftest_access_item *tmp;
	460
	461	mutex_lock(&staccess->lock);
	462	list_for_each_entry_safe(item, tmp, &staccess->items, items_elm) {
	463	if (iova > item->iova + item->length - 1 \|\|
	464	iova_last < item->iova)
	465	continue;
	466	list_del(&item->items_elm);
	467	iommufd_access_unpin_pages(staccess->access, item->iova,
	468	item->length);
	469	kfree(item);
	470	}
	471	mutex_unlock(&staccess->lock);
	472	}
	473
	474	static int iommufd_test_access_item_destroy(struct iommufd_ucmd *ucmd,
	475	unsigned int access_id,
	476	unsigned int item_id)
	477	{
	478	struct selftest_access_item *item;
	479	struct selftest_access *staccess;
	480
	481	staccess = iommufd_access_get(access_id);
	482	if (IS_ERR(staccess))
	483	return PTR_ERR(staccess);
	484
	485	mutex_lock(&staccess->lock);
	486	list_for_each_entry(item, &staccess->items, items_elm) {
	487	if (item->id == item_id) {
	488	list_del(&item->items_elm);
	489	iommufd_access_unpin_pages(staccess->access, item->iova,
	490	item->length);
	491	mutex_unlock(&staccess->lock);
	492	kfree(item);
	493	fput(staccess->file);
	494	return 0;
	495	}
	496	}
	497	mutex_unlock(&staccess->lock);
	498	fput(staccess->file);
	499	return -ENOENT;
	500	}
	501
	502	static int iommufd_test_staccess_release(struct inode *inode,
	503	struct file *filep)
	504	{
	505	struct selftest_access *staccess = filep->private_data;
	506
	507	if (staccess->access) {
	508	iommufd_test_access_unmap(staccess, 0, ULONG_MAX);
	509	iommufd_access_destroy(staccess->access);
	510	}
	511	mutex_destroy(&staccess->lock);
	512	kfree(staccess);
	513	return 0;
	514	}
	515
	516	static const struct iommufd_access_ops selftest_access_ops_pin = {
	517	.needs_pin_pages = 1,
	518	.unmap = iommufd_test_access_unmap,
	519	};
	520
	521	static const struct iommufd_access_ops selftest_access_ops = {
	522	.unmap = iommufd_test_access_unmap,
	523	};
	524
	525	static const struct file_operations iommfd_test_staccess_fops = {
	526	.release = iommufd_test_staccess_release,
	527	};
	528
	529	static struct selftest_access *iommufd_test_alloc_access(void)
	530	{
	531	struct selftest_access *staccess;
	532	struct file *filep;
	533
	534	staccess = kzalloc(sizeof(*staccess), GFP_KERNEL_ACCOUNT);
	535	if (!staccess)
	536	return ERR_PTR(-ENOMEM);
	537	INIT_LIST_HEAD(&staccess->items);
	538	mutex_init(&staccess->lock);
	539
	540	filep = anon_inode_getfile("[iommufd_test_staccess]",
	541	&iommfd_test_staccess_fops, staccess,
	542	O_RDWR);
	543	if (IS_ERR(filep)) {
	544	kfree(staccess);
	545	return ERR_CAST(filep);
	546	}
	547	staccess->file = filep;
	548	return staccess;
	549	}
	550
	551	static int iommufd_test_create_access(struct iommufd_ucmd *ucmd,
	552	unsigned int ioas_id, unsigned int flags)
	553	{
	554	struct iommu_test_cmd *cmd = ucmd->cmd;
	555	struct selftest_access *staccess;
	556	struct iommufd_access *access;
	557	int fdno;
	558	int rc;
	559
	560	if (flags & ~MOCK_FLAGS_ACCESS_CREATE_NEEDS_PIN_PAGES)
	561	return -EOPNOTSUPP;
	562
	563	staccess = iommufd_test_alloc_access();
	564	if (IS_ERR(staccess))
	565	return PTR_ERR(staccess);
	566
	567	fdno = get_unused_fd_flags(O_CLOEXEC);
	568	if (fdno < 0) {
	569	rc = -ENOMEM;
	570	goto out_free_staccess;
	571	}
	572
	573	access = iommufd_access_create(
	574	ucmd->ictx, ioas_id,
	575	(flags & MOCK_FLAGS_ACCESS_CREATE_NEEDS_PIN_PAGES) ?
	576	&selftest_access_ops_pin :
	577	&selftest_access_ops,
	578	staccess);
	579	if (IS_ERR(access)) {
	580	rc = PTR_ERR(access);
	581	goto out_put_fdno;
	582	}
	583	cmd->create_access.out_access_fd = fdno;
	584	rc = iommufd_ucmd_respond(ucmd, sizeof(*cmd));
	585	if (rc)
	586	goto out_destroy;
	587
	588	staccess->access = access;
	589	fd_install(fdno, staccess->file);
	590	return 0;
	591
	592	out_destroy:
	593	iommufd_access_destroy(access);
	594	out_put_fdno:
	595	put_unused_fd(fdno);
	596	out_free_staccess:
	597	fput(staccess->file);
	598	return rc;
	599	}
	600
	601	/* Check that the pages in a page array match the pages in the user VA */
	602	static int iommufd_test_check_pages(void __user uptr, struct page *pages,
	603	size_t npages)
	604	{
	605	for (; npages; npages--) {
	606	struct page *tmp_pages[1];
	607	long rc;
	608
	609	rc = get_user_pages_fast((uintptr_t)uptr, 1, 0, tmp_pages);
	610	if (rc < 0)
	611	return rc;
	612	if (WARN_ON(rc != 1))
	613	return -EFAULT;
	614	put_page(tmp_pages[0]);
	615	if (tmp_pages[0] != *pages)
	616	return -EBADE;
	617	pages++;
	618	uptr += PAGE_SIZE;
	619	}
	620	return 0;
	621	}
	622
	623	static int iommufd_test_access_pages(struct iommufd_ucmd *ucmd,
	624	unsigned int access_id, unsigned long iova,
	625	size_t length, void __user *uptr,
	626	u32 flags)
	627	{
	628	struct iommu_test_cmd *cmd = ucmd->cmd;
	629	struct selftest_access_item *item;
	630	struct selftest_access *staccess;
	631	struct page **pages;
	632	size_t npages;
	633	int rc;
	634
	635	/* Prevent syzkaller from triggering a WARN_ON in kvzalloc() */
	636	if (length > 1610241024)
	637	return -ENOMEM;
	638
	639	if (flags & ~(MOCK_FLAGS_ACCESS_WRITE \| MOCK_FLAGS_ACCESS_SYZ))
	640	return -EOPNOTSUPP;
	641
	642	staccess = iommufd_access_get(access_id);
	643	if (IS_ERR(staccess))
	644	return PTR_ERR(staccess);
	645
	646	if (staccess->access->ops != &selftest_access_ops_pin) {
	647	rc = -EOPNOTSUPP;
	648	goto out_put;
	649	}
	650
	651	if (flags & MOCK_FLAGS_ACCESS_SYZ)
	652	iova = iommufd_test_syz_conv_iova(&staccess->access->ioas->iopt,
	653	&cmd->access_pages.iova);
	654
	655	npages = (ALIGN(iova + length, PAGE_SIZE) -
	656	ALIGN_DOWN(iova, PAGE_SIZE)) /
	657	PAGE_SIZE;
	658	pages = kvcalloc(npages, sizeof(*pages), GFP_KERNEL_ACCOUNT);
	659	if (!pages) {
	660	rc = -ENOMEM;
	661	goto out_put;
	662	}
	663
	664	/*
	665	* Drivers will need to think very carefully about this locking. The
	666	* core code can do multiple unmaps instantaneously after
	667	* iommufd_access_pin_pages() and all the unmaps must not return until
	668	* the range is unpinned. This simple implementation puts a global lock
	669	* around the pin, which may not suit drivers that want this to be a
	670	* performance path. drivers that get this wrong will trigger WARN_ON
	671	* races and cause EDEADLOCK failures to userspace.
	672	*/
	673	mutex_lock(&staccess->lock);
	674	rc = iommufd_access_pin_pages(staccess->access, iova, length, pages,
	675	flags & MOCK_FLAGS_ACCESS_WRITE);
	676	if (rc)
	677	goto out_unlock;
	678
	679	/* For syzkaller allow uptr to be NULL to skip this check */
	680	if (uptr) {
	681	rc = iommufd_test_check_pages(
	682	uptr - (iova - ALIGN_DOWN(iova, PAGE_SIZE)), pages,
	683	npages);
	684	if (rc)
	685	goto out_unaccess;
	686	}
	687
	688	item = kzalloc(sizeof(*item), GFP_KERNEL_ACCOUNT);
	689	if (!item) {
	690	rc = -ENOMEM;
	691	goto out_unaccess;
	692	}
	693
	694	item->iova = iova;
	695	item->length = length;
	696	item->id = staccess->next_id++;
	697	list_add_tail(&item->items_elm, &staccess->items);
	698
	699	cmd->access_pages.out_access_pages_id = item->id;
	700	rc = iommufd_ucmd_respond(ucmd, sizeof(*cmd));
	701	if (rc)
	702	goto out_free_item;
	703	goto out_unlock;
	704
	705	out_free_item:
	706	list_del(&item->items_elm);
	707	kfree(item);
	708	out_unaccess:
	709	iommufd_access_unpin_pages(staccess->access, iova, length);
	710	out_unlock:
	711	mutex_unlock(&staccess->lock);
	712	kvfree(pages);
	713	out_put:
	714	fput(staccess->file);
	715	return rc;
	716	}
	717
	718	static int iommufd_test_access_rw(struct iommufd_ucmd *ucmd,
	719	unsigned int access_id, unsigned long iova,
	720	size_t length, void __user *ubuf,
	721	unsigned int flags)
	722	{
	723	struct iommu_test_cmd *cmd = ucmd->cmd;
	724	struct selftest_access *staccess;
	725	void *tmp;
	726	int rc;
	727
	728	/* Prevent syzkaller from triggering a WARN_ON in kvzalloc() */
	729	if (length > 1610241024)
	730	return -ENOMEM;
	731
	732	if (flags & ~(MOCK_ACCESS_RW_WRITE \| MOCK_ACCESS_RW_SLOW_PATH \|
	733	MOCK_FLAGS_ACCESS_SYZ))
	734	return -EOPNOTSUPP;
	735
	736	staccess = iommufd_access_get(access_id);
	737	if (IS_ERR(staccess))
	738	return PTR_ERR(staccess);
	739
	740	tmp = kvzalloc(length, GFP_KERNEL_ACCOUNT);
	741	if (!tmp) {
	742	rc = -ENOMEM;
	743	goto out_put;
	744	}
	745
	746	if (flags & MOCK_ACCESS_RW_WRITE) {
	747	if (copy_from_user(tmp, ubuf, length)) {
	748	rc = -EFAULT;
	749	goto out_free;
	750	}
	751	}
	752
	753	if (flags & MOCK_FLAGS_ACCESS_SYZ)
	754	iova = iommufd_test_syz_conv_iova(&staccess->access->ioas->iopt,
	755	&cmd->access_rw.iova);
	756
	757	rc = iommufd_access_rw(staccess->access, iova, tmp, length, flags);
	758	if (rc)
	759	goto out_free;
	760	if (!(flags & MOCK_ACCESS_RW_WRITE)) {
	761	if (copy_to_user(ubuf, tmp, length)) {
	762	rc = -EFAULT;
	763	goto out_free;
	764	}
	765	}
	766
	767	out_free:
	768	kvfree(tmp);
	769	out_put:
	770	fput(staccess->file);
	771	return rc;
	772	}
	773	static_assert((unsigned int)MOCK_ACCESS_RW_WRITE == IOMMUFD_ACCESS_RW_WRITE);
	774	static_assert((unsigned int)MOCK_ACCESS_RW_SLOW_PATH ==
	775	__IOMMUFD_ACCESS_RW_SLOW_PATH);
	776
	777	void iommufd_selftest_destroy(struct iommufd_object *obj)
	778	{
	779	struct selftest_obj *sobj = container_of(obj, struct selftest_obj, obj);
	780
	781	switch (sobj->type) {
	782	case TYPE_IDEV:
	783	iommufd_device_selftest_detach(sobj->idev.ictx,
	784	sobj->idev.hwpt);
	785	break;
	786	}
	787	}
	788
	789	int iommufd_test(struct iommufd_ucmd *ucmd)
	790	{
	791	struct iommu_test_cmd *cmd = ucmd->cmd;
	792
	793	switch (cmd->op) {
	794	case IOMMU_TEST_OP_ADD_RESERVED:
	795	return iommufd_test_add_reserved(ucmd, cmd->id,
	796	cmd->add_reserved.start,
	797	cmd->add_reserved.length);
	798	case IOMMU_TEST_OP_MOCK_DOMAIN:
	799	return iommufd_test_mock_domain(ucmd, cmd);
	800	case IOMMU_TEST_OP_MD_CHECK_MAP:
	801	return iommufd_test_md_check_pa(
	802	ucmd, cmd->id, cmd->check_map.iova,
	803	cmd->check_map.length,
	804	u64_to_user_ptr(cmd->check_map.uptr));
	805	case IOMMU_TEST_OP_MD_CHECK_REFS:
	806	return iommufd_test_md_check_refs(
	807	ucmd, u64_to_user_ptr(cmd->check_refs.uptr),
	808	cmd->check_refs.length, cmd->check_refs.refs);
	809	case IOMMU_TEST_OP_CREATE_ACCESS:
	810	return iommufd_test_create_access(ucmd, cmd->id,
	811	cmd->create_access.flags);
	812	case IOMMU_TEST_OP_ACCESS_PAGES:
	813	return iommufd_test_access_pages(
	814	ucmd, cmd->id, cmd->access_pages.iova,
	815	cmd->access_pages.length,
	816	u64_to_user_ptr(cmd->access_pages.uptr),
	817	cmd->access_pages.flags);
	818	case IOMMU_TEST_OP_ACCESS_RW:
	819	return iommufd_test_access_rw(
	820	ucmd, cmd->id, cmd->access_rw.iova,
	821	cmd->access_rw.length,
	822	u64_to_user_ptr(cmd->access_rw.uptr),
	823	cmd->access_rw.flags);
	824	case IOMMU_TEST_OP_DESTROY_ACCESS_PAGES:
	825	return iommufd_test_access_item_destroy(
	826	ucmd, cmd->id, cmd->destroy_access_pages.access_pages_id);
	827	case IOMMU_TEST_OP_SET_TEMP_MEMORY_LIMIT:
	828	/* Protect _batch_init(), can not be less than elmsz */
	829	if (cmd->memory_limit.limit <
	830	sizeof(unsigned long) + sizeof(u32))
	831	return -EINVAL;
	832	iommufd_test_memory_limit = cmd->memory_limit.limit;
	833	return 0;
	834	default:
	835	return -EOPNOTSUPP;
	836	}
	837	}
	838
	839	bool iommufd_should_fail(void)
	840	{
	841	return should_fail(&fail_iommufd, 1);
	842	}
	843
	844	void __init iommufd_test_init(void)
	845	{
	846	dbgfs_root =
	847	fault_create_debugfs_attr("fail_iommufd", NULL, &fail_iommufd);
	848	}
	849
	850	void iommufd_test_exit(void)
	851	{
	852	debugfs_remove_recursive(dbgfs_root);
	853	}

drivers/iommu/iommufd/vfio_compat.c+472−0

@@ -0,0 +1,472 @@
	1	// SPDX-License-Identifier: GPL-2.0-only
	2	/* Copyright (c) 2021-2022, NVIDIA CORPORATION & AFFILIATES
	3	*/
	4	#include <linux/file.h>
	5	#include <linux/interval_tree.h>
	6	#include <linux/iommu.h>
	7	#include <linux/iommufd.h>
	8	#include <linux/slab.h>
	9	#include <linux/vfio.h>
	10	#include <uapi/linux/vfio.h>
	11	#include <uapi/linux/iommufd.h>
	12
	13	#include "iommufd_private.h"
	14
	15	static struct iommufd_ioas get_compat_ioas(struct iommufd_ctx ictx)
	16	{
	17	struct iommufd_ioas *ioas = ERR_PTR(-ENODEV);
	18
	19	xa_lock(&ictx->objects);
	20	if (!ictx->vfio_ioas \|\| !iommufd_lock_obj(&ictx->vfio_ioas->obj))
	21	goto out_unlock;
	22	ioas = ictx->vfio_ioas;
	23	out_unlock:
	24	xa_unlock(&ictx->objects);
	25	return ioas;
	26	}
	27
	28	/**
	29	* iommufd_vfio_compat_ioas_id - Return the IOAS ID that vfio should use
	30	* @ictx: Context to operate on
	31	* @out_ioas_id: The ioas_id the caller should use
	32	*
	33	* The compatibility IOAS is the IOAS that the vfio compatibility ioctls operate
	34	* on since they do not have an IOAS ID input in their ABI. Only attaching a
	35	* group should cause a default creation of the internal ioas, this returns the
	36	* existing ioas if it has already been assigned somehow.
	37	*/
	38	int iommufd_vfio_compat_ioas_id(struct iommufd_ctx ictx, u32 out_ioas_id)
	39	{
	40	struct iommufd_ioas *ioas = NULL;
	41	struct iommufd_ioas *out_ioas;
	42
	43	ioas = iommufd_ioas_alloc(ictx);
	44	if (IS_ERR(ioas))
	45	return PTR_ERR(ioas);
	46
	47	xa_lock(&ictx->objects);
	48	if (ictx->vfio_ioas && iommufd_lock_obj(&ictx->vfio_ioas->obj))
	49	out_ioas = ictx->vfio_ioas;
	50	else {
	51	out_ioas = ioas;
	52	ictx->vfio_ioas = ioas;
	53	}
	54	xa_unlock(&ictx->objects);
	55
	56	*out_ioas_id = out_ioas->obj.id;
	57	if (out_ioas != ioas) {
	58	iommufd_put_object(&out_ioas->obj);
	59	iommufd_object_abort(ictx, &ioas->obj);
	60	return 0;
	61	}
	62	/*
	63	* An automatically created compat IOAS is treated as a userspace
	64	* created object. Userspace can learn the ID via IOMMU_VFIO_IOAS_GET,
	65	* and if not manually destroyed it will be destroyed automatically
	66	* at iommufd release.
	67	*/
	68	iommufd_object_finalize(ictx, &ioas->obj);
	69	return 0;
	70	}
	71	EXPORT_SYMBOL_NS_GPL(iommufd_vfio_compat_ioas_id, IOMMUFD_VFIO);
	72
	73	int iommufd_vfio_ioas(struct iommufd_ucmd *ucmd)
	74	{
	75	struct iommu_vfio_ioas *cmd = ucmd->cmd;
	76	struct iommufd_ioas *ioas;
	77
	78	if (cmd->__reserved)
	79	return -EOPNOTSUPP;
	80	switch (cmd->op) {
	81	case IOMMU_VFIO_IOAS_GET:
	82	ioas = get_compat_ioas(ucmd->ictx);
	83	if (IS_ERR(ioas))
	84	return PTR_ERR(ioas);
	85	cmd->ioas_id = ioas->obj.id;
	86	iommufd_put_object(&ioas->obj);
	87	return iommufd_ucmd_respond(ucmd, sizeof(*cmd));
	88
	89	case IOMMU_VFIO_IOAS_SET:
	90	ioas = iommufd_get_ioas(ucmd, cmd->ioas_id);
	91	if (IS_ERR(ioas))
	92	return PTR_ERR(ioas);
	93	xa_lock(&ucmd->ictx->objects);
	94	ucmd->ictx->vfio_ioas = ioas;
	95	xa_unlock(&ucmd->ictx->objects);
	96	iommufd_put_object(&ioas->obj);
	97	return 0;
	98
	99	case IOMMU_VFIO_IOAS_CLEAR:
	100	xa_lock(&ucmd->ictx->objects);
	101	ucmd->ictx->vfio_ioas = NULL;
	102	xa_unlock(&ucmd->ictx->objects);
	103	return 0;
	104	default:
	105	return -EOPNOTSUPP;
	106	}
	107	}
	108
	109	static int iommufd_vfio_map_dma(struct iommufd_ctx *ictx, unsigned int cmd,
	110	void __user *arg)
	111	{
	112	u32 supported_flags = VFIO_DMA_MAP_FLAG_READ \| VFIO_DMA_MAP_FLAG_WRITE;
	113	size_t minsz = offsetofend(struct vfio_iommu_type1_dma_map, size);
	114	struct vfio_iommu_type1_dma_map map;
	115	int iommu_prot = IOMMU_CACHE;
	116	struct iommufd_ioas *ioas;
	117	unsigned long iova;
	118	int rc;
	119
	120	if (copy_from_user(&map, arg, minsz))
	121	return -EFAULT;
	122
	123	if (map.argsz < minsz \|\| map.flags & ~supported_flags)
	124	return -EINVAL;
	125
	126	if (map.flags & VFIO_DMA_MAP_FLAG_READ)
	127	iommu_prot \|= IOMMU_READ;
	128	if (map.flags & VFIO_DMA_MAP_FLAG_WRITE)
	129	iommu_prot \|= IOMMU_WRITE;
	130
	131	ioas = get_compat_ioas(ictx);
	132	if (IS_ERR(ioas))
	133	return PTR_ERR(ioas);
	134
	135	/*
	136	* Maps created through the legacy interface always use VFIO compatible
	137	* rlimit accounting. If the user wishes to use the faster user based
	138	* rlimit accounting then they must use the new interface.
	139	*/
	140	iova = map.iova;
	141	rc = iopt_map_user_pages(ictx, &ioas->iopt, &iova, u64_to_user_ptr(map.vaddr),
	142	map.size, iommu_prot, 0);
	143	iommufd_put_object(&ioas->obj);
	144	return rc;
	145	}
	146
	147	static int iommufd_vfio_unmap_dma(struct iommufd_ctx *ictx, unsigned int cmd,
	148	void __user *arg)
	149	{
	150	size_t minsz = offsetofend(struct vfio_iommu_type1_dma_unmap, size);
	151	/*
	152	* VFIO_DMA_UNMAP_FLAG_GET_DIRTY_BITMAP is obsoleted by the new
	153	* dirty tracking direction:
	154	* https://lore.kernel.org/kvm/20220731125503.142683-1-yishaih@nvidia.com/
	155	* https://lore.kernel.org/kvm/20220428210933.3583-1-joao.m.martins@oracle.com/
	156	*/
	157	u32 supported_flags = VFIO_DMA_UNMAP_FLAG_ALL;
	158	struct vfio_iommu_type1_dma_unmap unmap;
	159	unsigned long unmapped = 0;
	160	struct iommufd_ioas *ioas;
	161	int rc;
	162
	163	if (copy_from_user(&unmap, arg, minsz))
	164	return -EFAULT;
	165
	166	if (unmap.argsz < minsz \|\| unmap.flags & ~supported_flags)
	167	return -EINVAL;
	168
	169	ioas = get_compat_ioas(ictx);
	170	if (IS_ERR(ioas))
	171	return PTR_ERR(ioas);
	172
	173	if (unmap.flags & VFIO_DMA_UNMAP_FLAG_ALL) {
	174	if (unmap.iova != 0 \|\| unmap.size != 0) {
	175	rc = -EINVAL;
	176	goto err_put;
	177	}
	178	rc = iopt_unmap_all(&ioas->iopt, &unmapped);
	179	} else {
	180	if (READ_ONCE(ioas->iopt.disable_large_pages)) {
	181	/*
	182	* Create cuts at the start and last of the requested
	183	* range. If the start IOVA is 0 then it doesn't need to
	184	* be cut.
	185	*/
	186	unsigned long iovas[] = { unmap.iova + unmap.size - 1,
	187	unmap.iova - 1 };
	188
	189	rc = iopt_cut_iova(&ioas->iopt, iovas,
	190	unmap.iova ? 2 : 1);
	191	if (rc)
	192	goto err_put;
	193	}
	194	rc = iopt_unmap_iova(&ioas->iopt, unmap.iova, unmap.size,
	195	&unmapped);
	196	}
	197	unmap.size = unmapped;
	198	if (copy_to_user(arg, &unmap, minsz))
	199	rc = -EFAULT;
	200
	201	err_put:
	202	iommufd_put_object(&ioas->obj);
	203	return rc;
	204	}
	205
	206	static int iommufd_vfio_cc_iommu(struct iommufd_ctx *ictx)
	207	{
	208	struct iommufd_hw_pagetable *hwpt;
	209	struct iommufd_ioas *ioas;
	210	int rc = 1;
	211
	212	ioas = get_compat_ioas(ictx);
	213	if (IS_ERR(ioas))
	214	return PTR_ERR(ioas);
	215
	216	mutex_lock(&ioas->mutex);
	217	list_for_each_entry(hwpt, &ioas->hwpt_list, hwpt_item) {
	218	if (!hwpt->enforce_cache_coherency) {
	219	rc = 0;
	220	break;
	221	}
	222	}
	223	mutex_unlock(&ioas->mutex);
	224
	225	iommufd_put_object(&ioas->obj);
	226	return rc;
	227	}
	228
	229	static int iommufd_vfio_check_extension(struct iommufd_ctx *ictx,
	230	unsigned long type)
	231	{
	232	switch (type) {
	233	case VFIO_TYPE1_IOMMU:
	234	case VFIO_TYPE1v2_IOMMU:
	235	case VFIO_UNMAP_ALL:
	236	return 1;
	237
	238	case VFIO_DMA_CC_IOMMU:
	239	return iommufd_vfio_cc_iommu(ictx);
	240
	241	/*
	242	* This is obsolete, and to be removed from VFIO. It was an incomplete
	243	* idea that got merged.
	244	* https://lore.kernel.org/kvm/0-v1-0093c9b0e345+19-vfio_no_nesting_jgg@nvidia.com/
	245	*/
	246	case VFIO_TYPE1_NESTING_IOMMU:
	247	return 0;
	248
	249	/*
	250	* VFIO_DMA_MAP_FLAG_VADDR
	251	* https://lore.kernel.org/kvm/1611939252-7240-1-git-send-email-steven.sistare@oracle.com/
	252	* https://lore.kernel.org/all/Yz777bJZjTyLrHEQ@nvidia.com/
	253	*
	254	* It is hard to see how this could be implemented safely.
	255	*/
	256	case VFIO_UPDATE_VADDR:
	257	default:
	258	return 0;
	259	}
	260	}
	261
	262	static int iommufd_vfio_set_iommu(struct iommufd_ctx *ictx, unsigned long type)
	263	{
	264	struct iommufd_ioas *ioas = NULL;
	265	int rc = 0;
	266
	267	if (type != VFIO_TYPE1_IOMMU && type != VFIO_TYPE1v2_IOMMU)
	268	return -EINVAL;
	269
	270	/* VFIO fails the set_iommu if there is no group */
	271	ioas = get_compat_ioas(ictx);
	272	if (IS_ERR(ioas))
	273	return PTR_ERR(ioas);
	274
	275	/*
	276	* The difference between TYPE1 and TYPE1v2 is the ability to unmap in
	277	* the middle of mapped ranges. This is complicated by huge page support
	278	* which creates single large IOPTEs that cannot be split by the iommu
	279	* driver. TYPE1 is very old at this point and likely nothing uses it,
	280	* however it is simple enough to emulate by simply disabling the
	281	* problematic large IOPTEs. Then we can safely unmap within any range.
	282	*/
	283	if (type == VFIO_TYPE1_IOMMU)
	284	rc = iopt_disable_large_pages(&ioas->iopt);
	285	iommufd_put_object(&ioas->obj);
	286	return rc;
	287	}
	288
	289	static unsigned long iommufd_get_pagesizes(struct iommufd_ioas *ioas)
	290	{
	291	struct io_pagetable *iopt = &ioas->iopt;
	292	unsigned long pgsize_bitmap = ULONG_MAX;
	293	struct iommu_domain *domain;
	294	unsigned long index;
	295
	296	down_read(&iopt->domains_rwsem);
	297	xa_for_each(&iopt->domains, index, domain)
	298	pgsize_bitmap &= domain->pgsize_bitmap;
	299
	300	/* See vfio_update_pgsize_bitmap() */
	301	if (pgsize_bitmap & ~PAGE_MASK) {
	302	pgsize_bitmap &= PAGE_MASK;
	303	pgsize_bitmap \|= PAGE_SIZE;
	304	}
	305	pgsize_bitmap = max(pgsize_bitmap, ioas->iopt.iova_alignment);
	306	up_read(&iopt->domains_rwsem);
	307	return pgsize_bitmap;
	308	}
	309
	310	static int iommufd_fill_cap_iova(struct iommufd_ioas *ioas,
	311	struct vfio_info_cap_header __user *cur,
	312	size_t avail)
	313	{
	314	struct vfio_iommu_type1_info_cap_iova_range __user *ucap_iovas =
	315	container_of(cur,
	316	struct vfio_iommu_type1_info_cap_iova_range __user,
	317	header);
	318	struct vfio_iommu_type1_info_cap_iova_range cap_iovas = {
	319	.header = {
	320	.id = VFIO_IOMMU_TYPE1_INFO_CAP_IOVA_RANGE,
	321	.version = 1,
	322	},
	323	};
	324	struct interval_tree_span_iter span;
	325
	326	interval_tree_for_each_span(&span, &ioas->iopt.reserved_itree, 0,
	327	ULONG_MAX) {
	328	struct vfio_iova_range range;
	329
	330	if (!span.is_hole)
	331	continue;
	332	range.start = span.start_hole;
	333	range.end = span.last_hole;
	334	if (avail >= struct_size(&cap_iovas, iova_ranges,
	335	cap_iovas.nr_iovas + 1) &&
	336	copy_to_user(&ucap_iovas->iova_ranges[cap_iovas.nr_iovas],
	337	&range, sizeof(range)))
	338	return -EFAULT;
	339	cap_iovas.nr_iovas++;
	340	}
	341	if (avail >= struct_size(&cap_iovas, iova_ranges, cap_iovas.nr_iovas) &&
	342	copy_to_user(ucap_iovas, &cap_iovas, sizeof(cap_iovas)))
	343	return -EFAULT;
	344	return struct_size(&cap_iovas, iova_ranges, cap_iovas.nr_iovas);
	345	}
	346
	347	static int iommufd_fill_cap_dma_avail(struct iommufd_ioas *ioas,
	348	struct vfio_info_cap_header __user *cur,
	349	size_t avail)
	350	{
	351	struct vfio_iommu_type1_info_dma_avail cap_dma = {
	352	.header = {
	353	.id = VFIO_IOMMU_TYPE1_INFO_DMA_AVAIL,
	354	.version = 1,
	355	},
	356	/*
	357	* iommufd's limit is based on the cgroup's memory limit.
	358	* Normally vfio would return U16_MAX here, and provide a module
	359	* parameter to adjust it. Since S390 qemu userspace actually
	360	* pays attention and needs a value bigger than U16_MAX return
	361	* U32_MAX.
	362	*/
	363	.avail = U32_MAX,
	364	};
	365
	366	if (avail >= sizeof(cap_dma) &&
	367	copy_to_user(cur, &cap_dma, sizeof(cap_dma)))
	368	return -EFAULT;
	369	return sizeof(cap_dma);
	370	}
	371
	372	static int iommufd_vfio_iommu_get_info(struct iommufd_ctx *ictx,
	373	void __user *arg)
	374	{
	375	typedef int (fill_cap_fn)(struct iommufd_ioas ioas,
	376	struct vfio_info_cap_header __user *cur,
	377	size_t avail);
	378	static const fill_cap_fn fill_fns[] = {
	379	iommufd_fill_cap_dma_avail,
	380	iommufd_fill_cap_iova,
	381	};
	382	size_t minsz = offsetofend(struct vfio_iommu_type1_info, iova_pgsizes);
	383	struct vfio_info_cap_header __user *last_cap = NULL;
	384	struct vfio_iommu_type1_info info;
	385	struct iommufd_ioas *ioas;
	386	size_t total_cap_size;
	387	int rc;
	388	int i;
	389
	390	if (copy_from_user(&info, arg, minsz))
	391	return -EFAULT;
	392
	393	if (info.argsz < minsz)
	394	return -EINVAL;
	395	minsz = min_t(size_t, info.argsz, sizeof(info));
	396
	397	ioas = get_compat_ioas(ictx);
	398	if (IS_ERR(ioas))
	399	return PTR_ERR(ioas);
	400
	401	info.flags = VFIO_IOMMU_INFO_PGSIZES;
	402	info.iova_pgsizes = iommufd_get_pagesizes(ioas);
	403	info.cap_offset = 0;
	404
	405	down_read(&ioas->iopt.iova_rwsem);
	406	total_cap_size = sizeof(info);
	407	for (i = 0; i != ARRAY_SIZE(fill_fns); i++) {
	408	int cap_size;
	409
	410	if (info.argsz > total_cap_size)
	411	cap_size = fill_fns[i](ioas, arg + total_cap_size,
	412	info.argsz - total_cap_size);
	413	else
	414	cap_size = fill_fns[i](ioas, NULL, 0);
	415	if (cap_size < 0) {
	416	rc = cap_size;
	417	goto out_put;
	418	}
	419	if (last_cap && info.argsz >= total_cap_size &&
	420	put_user(total_cap_size, &last_cap->next)) {
	421	rc = -EFAULT;
	422	goto out_put;
	423	}
	424	last_cap = arg + total_cap_size;
	425	total_cap_size += cap_size;
	426	}
	427
	428	/*
	429	* If the user did not provide enough space then only some caps are
	430	* returned and the argsz will be updated to the correct amount to get
	431	* all caps.
	432	*/
	433	if (info.argsz >= total_cap_size)
	434	info.cap_offset = sizeof(info);
	435	info.argsz = total_cap_size;
	436	info.flags \|= VFIO_IOMMU_INFO_CAPS;
	437	if (copy_to_user(arg, &info, minsz)) {
	438	rc = -EFAULT;
	439	goto out_put;
	440	}
	441	rc = 0;
	442
	443	out_put:
	444	up_read(&ioas->iopt.iova_rwsem);
	445	iommufd_put_object(&ioas->obj);
	446	return rc;
	447	}
	448
	449	int iommufd_vfio_ioctl(struct iommufd_ctx *ictx, unsigned int cmd,
	450	unsigned long arg)
	451	{
	452	void __user uarg = (void __user )arg;
	453
	454	switch (cmd) {
	455	case VFIO_GET_API_VERSION:
	456	return VFIO_API_VERSION;
	457	case VFIO_SET_IOMMU:
	458	return iommufd_vfio_set_iommu(ictx, arg);
	459	case VFIO_CHECK_EXTENSION:
	460	return iommufd_vfio_check_extension(ictx, arg);
	461	case VFIO_IOMMU_GET_INFO:
	462	return iommufd_vfio_iommu_get_info(ictx, uarg);
	463	case VFIO_IOMMU_MAP_DMA:
	464	return iommufd_vfio_map_dma(ictx, cmd, uarg);
	465	case VFIO_IOMMU_UNMAP_DMA:
	466	return iommufd_vfio_unmap_dma(ictx, cmd, uarg);
	467	case VFIO_IOMMU_DIRTY_PAGES:
	468	default:
	469	return -ENOIOCTLCMD;
	470	}
	471	return -ENOIOCTLCMD;
	472	}

drivers/iommu/ipmmu-vmsa.c+0−2

 		 * Something is wrong, we can't attach two devices using
 		 * different IOMMUs to the same domain.
 		 */
-		dev_err(dev, "Can't attach IPMMU %s to domain on IPMMU %s\n",
-			dev_name(mmu->dev), dev_name(domain->mmu->dev));
 		ret = -EINVAL;
 	} else
 		dev_info(dev, "Reusing IPMMU context %u\n", domain->context_id);

drivers/iommu/mtk_iommu.c+2−2

@@ -609,7 +609,7 @@ static int mtk_iommu_domain_finalise(struct mtk_iommu_domain *dom, · 608 unmodified lines
609	dom->iop = alloc_io_pgtable_ops(ARM_V7S, &dom->cfg, data);	609	dom->iop = alloc_io_pgtable_ops(ARM_V7S, &dom->cfg, data);
610	if (!dom->iop) {	610	if (!dom->iop) {
611	dev_err(data->dev, "Failed to alloc io pgtable\n");	611	dev_err(data->dev, "Failed to alloc io pgtable\n");
612	return -EINVAL;	612	return -ENOMEM;
613	}	613	}
614		614
615	/* Update our support page sizes bitmap */	615	/* Update our support page sizes bitmap */
@@ -668,7 +668,7 @@ static int mtk_iommu_attach_device(struct iommu_domain *domain, · 52 unmodified lines
668	ret = mtk_iommu_domain_finalise(dom, frstdata, region_id);	668	ret = mtk_iommu_domain_finalise(dom, frstdata, region_id);
669	if (ret) {	669	if (ret) {
670	mutex_unlock(&dom->mutex);	670	mutex_unlock(&dom->mutex);
671	return -ENODEV;	671	return ret;
672	}	672	}
673	dom->bank = &data->bank[bankid];	673	dom->bank = &data->bank[bankid];
674	}	674	}

drivers/iommu/omap-iommu.c+3−3

@@ -1414,7 +1414,7 @@ static int omap_iommu_attach_init(struct device *dev, · 1413 unmodified lines
1414		1414
1415	odomain->num_iommus = omap_iommu_count(dev);	1415	odomain->num_iommus = omap_iommu_count(dev);
1416	if (!odomain->num_iommus)	1416	if (!odomain->num_iommus)
1417	return -EINVAL;	1417	return -ENODEV;
1418		1418
1419	odomain->iommus = kcalloc(odomain->num_iommus, sizeof(*iommu),	1419	odomain->iommus = kcalloc(odomain->num_iommus, sizeof(*iommu),
1420	GFP_ATOMIC);	1420	GFP_ATOMIC);
@@ -1464,7 +1464,7 @@ omap_iommu_attach_dev(struct iommu_domain domain, struct device dev) · 43 unmodified lines
1464		1464
1465	if (!arch_data \|\| !arch_data->iommu_dev) {	1465	if (!arch_data \|\| !arch_data->iommu_dev) {
1466	dev_err(dev, "device doesn't have an associated iommu\n");	1466	dev_err(dev, "device doesn't have an associated iommu\n");
1467	return -EINVAL;	1467	return -ENODEV;
1468	}	1468	}
1469		1469
1470	spin_lock(&omap_domain->lock);	1470	spin_lock(&omap_domain->lock);
@@ -1472,7 +1472,7 @@ omap_iommu_attach_dev(struct iommu_domain domain, struct device dev) · 1 unmodified line
1472	/* only a single client device can be attached to a domain */	1472	/* only a single client device can be attached to a domain */
1473	if (omap_domain->dev) {	1473	if (omap_domain->dev) {
1474	dev_err(dev, "iommu domain is already attached\n");	1474	dev_err(dev, "iommu domain is already attached\n");
1475	ret = -EBUSY;	1475	ret = -EINVAL;
1476	goto out;	1476	goto out;
1477	}	1477	}
1478		1478

drivers/iommu/sprd-iommu.c+1−3

 	struct sprd_iommu_domain *dom = to_sprd_domain(domain);
 	size_t pgt_size = sprd_iommu_pgt_size(domain);
-	if (dom->sdev) {
+	if (dom->sdev)
-		pr_err("There's already a device attached to this domain.\n");
 		return -EINVAL;
-	}
 	dom->pgt_va = dma_alloc_coherent(sdev->dev, pgt_size, &dom->pgt_pa, GFP_KERNEL);
 	if (!dom->pgt_va)

drivers/iommu/tegra-gart.c+1−1

@@ -112,7 +112,7 @@ static int gart_iommu_attach_dev(struct iommu_domain *domain, · 111 unmodified lines
112	spin_lock(&gart->dom_lock);	112	spin_lock(&gart->dom_lock);
113		113
114	if (gart->active_domain && gart->active_domain != domain) {	114	if (gart->active_domain && gart->active_domain != domain) {
115	ret = -EBUSY;	115	ret = -EINVAL;
116	} else if (dev_iommu_priv_get(dev) != domain) {	116	} else if (dev_iommu_priv_get(dev) != domain) {
117	dev_iommu_priv_set(dev, domain);	117	dev_iommu_priv_set(dev, domain);
118	gart->active_domain = domain;	118	gart->active_domain = domain;

drivers/iommu/virtio-iommu.c+3−4

 		dev_err(vdev->dev,
 			"granule 0x%lx larger than system page size 0x%lx\n",
 			viommu_page_size, PAGE_SIZE);
-		return -EINVAL;
+		return -ENODEV;
 	}
 	ret = ida_alloc_range(&viommu->domain_ids, viommu->first_domain,
 		if (ret) {
 			ida_free(&viommu->domain_ids, vdomain->id);
 			vdomain->viommu = NULL;
-			return -EOPNOTSUPP;
+			return ret;
 		}
 	}
 		 */
 		ret = viommu_domain_finalise(vdev, domain);
 	} else if (vdomain->viommu != vdev->viommu) {
-		dev_err(dev, "cannot attach to foreign vIOMMU\n");
+		ret = -EINVAL;
-		ret = -EXDEV;
 	}
 	mutex_unlock(&vdomain->mutex);

drivers/misc/uacce/uacce.c+1−1

@@ -108,7 +108,7 @@ static int uacce_bind_queue(struct uacce_device uacce, struct uacce_queue q) · 107 unmodified lines
108	if (!(uacce->flags & UACCE_DEV_SVA))	108	if (!(uacce->flags & UACCE_DEV_SVA))
109	return 0;	109	return 0;
110		110
111	handle = iommu_sva_bind_device(uacce->parent, current->mm, NULL);	111	handle = iommu_sva_bind_device(uacce->parent, current->mm);
112	if (IS_ERR(handle))	112	if (IS_ERR(handle))
113	return PTR_ERR(handle);	113	return PTR_ERR(handle);
114		114

drivers/pci/ats.c+3−0

@@ -382,6 +382,9 @@ int pci_enable_pasid(struct pci_dev *pdev, int features) · 381 unmodified lines
382	if (!pasid)	382	if (!pasid)
383	return -EINVAL;	383	return -EINVAL;
384		384
		385	if (!pci_acs_path_enabled(pdev, NULL, PCI_ACS_RR \| PCI_ACS_UF))
		386	return -EINVAL;
		387
385	pci_read_config_word(pdev, pasid + PCI_PASID_CAP, &supported);	388	pci_read_config_word(pdev, pasid + PCI_PASID_CAP, &supported);
386	supported &= PCI_PASID_CAP_EXEC \| PCI_PASID_CAP_PRIV;	389	supported &= PCI_PASID_CAP_EXEC \| PCI_PASID_CAP_PRIV;
387		390

drivers/s390/cio/vfio_ccw_ops.c+3−0

 	.ioctl = vfio_ccw_mdev_ioctl,
 	.request = vfio_ccw_mdev_request,
 	.dma_unmap = vfio_ccw_dma_unmap,
+	.bind_iommufd = vfio_iommufd_emulated_bind,
+	.unbind_iommufd = vfio_iommufd_emulated_unbind,
+	.attach_ioas = vfio_iommufd_emulated_attach_ioas,
 };
 struct mdev_driver vfio_ccw_mdev_driver = {

drivers/s390/crypto/vfio_ap_ops.c+20−1

 	return 0;
 }
+static void unmap_iova(struct ap_matrix_mdev *matrix_mdev, u64 iova, u64 length)
+{
+	struct ap_queue_table *qtable = &matrix_mdev->qtable;
+	struct vfio_ap_queue *q;
+	int loop_cursor;
+	hash_for_each(qtable->queues, loop_cursor, q, mdev_qnode) {
+		if (q->saved_iova >= iova && q->saved_iova < iova + length)
+			vfio_ap_irq_disable(q);
+	}
+}
 static void vfio_ap_mdev_dma_unmap(struct vfio_device *vdev, u64 iova,
 				   u64 length)
 {
 	struct ap_matrix_mdev *matrix_mdev =
 		container_of(vdev, struct ap_matrix_mdev, vdev);
-	vfio_unpin_pages(&matrix_mdev->vdev, iova, 1);
+	mutex_lock(&matrix_dev->mdevs_lock);
+	unmap_iova(matrix_mdev, iova, length);
+	mutex_unlock(&matrix_dev->mdevs_lock);
 }
 /**
 	.close_device = vfio_ap_mdev_close_device,
 	.ioctl = vfio_ap_mdev_ioctl,
 	.dma_unmap = vfio_ap_mdev_dma_unmap,
+	.bind_iommufd = vfio_iommufd_emulated_bind,
+	.unbind_iommufd = vfio_iommufd_emulated_unbind,
+	.attach_ioas = vfio_iommufd_emulated_attach_ioas,
 };
 static struct mdev_driver vfio_ap_matrix_driver = {

drivers/vfio/Kconfig+25−11

 menuconfig VFIO
 	tristate "VFIO Non-Privileged userspace driver framework"
 	select IOMMU_API
-	select VFIO_IOMMU_TYPE1 if MMU && (X86 || S390 || ARM || ARM64)
+	depends on IOMMUFD || !IOMMUFD
 	select INTERVAL_TREE
+	select VFIO_CONTAINER if IOMMUFD=n
 	help
 	  VFIO provides a framework for secure userspace device drivers.
 	  See Documentation/driver-api/vfio.rst for more details.
 	  If you don't know what to do here, say N.
 if VFIO
+config VFIO_CONTAINER
+	bool "Support for the VFIO container /dev/vfio/vfio"
+	select VFIO_IOMMU_TYPE1 if MMU && (X86 || S390 || ARM || ARM64)
+	default y
+	help
+	  The VFIO container is the classic interface to VFIO for establishing
+	  IOMMU mappings. If N is selected here then IOMMUFD must be used to
+	  manage the mappings.
+	  Unless testing IOMMUFD say Y here.
+if VFIO_CONTAINER
 config VFIO_IOMMU_TYPE1
 	tristate
 	default n
 	depends on SPAPR_TCE_IOMMU
 	default VFIO
-config VFIO_SPAPR_EEH
-	tristate
-	depends on EEH && VFIO_IOMMU_SPAPR_TCE
-	default VFIO
-config VFIO_VIRQFD
-	tristate
-	select EVENTFD
-	default n
 config VFIO_NOIOMMU
 	bool "VFIO No-IOMMU support"
 	help
 	  this mode since there is no IOMMU to provide DMA translation.
 	  If you don't know what to do here, say N.
+endif
+config VFIO_SPAPR_EEH
+	tristate
+	depends on EEH && VFIO_IOMMU_SPAPR_TCE
+	default VFIO
+config VFIO_VIRQFD
+	tristate
+	select EVENTFD
+	default n
 source "drivers/vfio/pci/Kconfig"
 source "drivers/vfio/platform/Kconfig"

drivers/vfio/Makefile+4−2

 obj-$(CONFIG_VFIO) += vfio.o
 vfio-y += vfio_main.o \
-	  iova_bitmap.o \
+	  group.o \
-	  container.o
+	  iova_bitmap.o
+vfio-$(CONFIG_IOMMUFD) += iommufd.o
+vfio-$(CONFIG_VFIO_CONTAINER) += container.o
 obj-$(CONFIG_VFIO_VIRQFD) += vfio_virqfd.o
 obj-$(CONFIG_VFIO_IOMMU_TYPE1) += vfio_iommu_type1.o

drivers/vfio/container.c+42−103

@@ -188,8 +188,9 @@ void vfio_device_container_unregister(struct vfio_device *device) · 187 unmodified lines
188	device->group->container->iommu_data, device);	188	device->group->container->iommu_data, device);
189	}	189	}
190		190
191	long vfio_container_ioctl_check_extension(struct vfio_container *container,	191	static long
192	unsigned long arg)	192	vfio_container_ioctl_check_extension(struct vfio_container *container,
		193	unsigned long arg)
193	{	194	{
194	struct vfio_iommu_driver *driver;	195	struct vfio_iommu_driver *driver;
195	long ret = 0;	196	long ret = 0;
@@ -511,14 +512,15 @@ void vfio_group_detach_container(struct vfio_group *group) · 315 unmodified lines
511	vfio_container_put(container);	512	vfio_container_put(container);
512	}	513	}
513		514
514	int vfio_device_assign_container(struct vfio_device *device)	515	int vfio_group_use_container(struct vfio_group *group)
515	{	516	{
516	struct vfio_group *group = device->group;
517
518	lockdep_assert_held(&group->group_lock);	517	lockdep_assert_held(&group->group_lock);
519		518
520	if (!group->container \|\| !group->container->iommu_driver \|\|	519	/*
521	WARN_ON(!group->container_users))	520	* The container fd has been assigned with VFIO_GROUP_SET_CONTAINER but
		521	* VFIO_SET_IOMMU hasn't been done yet.
		522	*/
		523	if (!group->container->iommu_driver)
522	return -EINVAL;	524	return -EINVAL;
523		525
524	if (group->type == VFIO_NO_IOMMU && !capable(CAP_SYS_RAWIO))	526	if (group->type == VFIO_NO_IOMMU && !capable(CAP_SYS_RAWIO))
@@ -529,122 +531,56 @@ int vfio_device_assign_container(struct vfio_device *device) · 4 unmodified lines
529	return 0;	531	return 0;
530	}	532	}
531		533
532	void vfio_device_unassign_container(struct vfio_device *device)	534	void vfio_group_unuse_container(struct vfio_group *group)
533	{	535	{
534	mutex_lock(&device->group->group_lock);	536	lockdep_assert_held(&group->group_lock);
535	WARN_ON(device->group->container_users <= 1);	537
536	device->group->container_users--;	538	WARN_ON(group->container_users <= 1);
537	fput(device->group->opened_file);	539	group->container_users--;
538	mutex_unlock(&device->group->group_lock);	540	fput(group->opened_file);
539	}	541	}
540		542
541	/*	543	int vfio_device_container_pin_pages(struct vfio_device *device,
542	* Pin contiguous user pages and return their associated host pages for local	544	dma_addr_t iova, int npage,
543	* domain only.	545	int prot, struct page **pages)
544	* @device [in] : device
545	* @iova [in] : starting IOVA of user pages to be pinned.
546	* @npage [in] : count of pages to be pinned. This count should not
547	* be greater than VFIO_PIN_PAGES_MAX_ENTRIES.
548	* @prot [in] : protection flags
549	* @pages[out] : array of host pages
550	* Return error or number of pages pinned.
551	*
552	* A driver may only call this function if the vfio_device was created
553	* by vfio_register_emulated_iommu_dev().
554	*/
555	int vfio_pin_pages(struct vfio_device *device, dma_addr_t iova,
556	int npage, int prot, struct page **pages)
557	{	546	{
558	struct vfio_container *container;	547	struct vfio_container *container = device->group->container;
559	struct vfio_group *group = device->group;	548	struct iommu_group *iommu_group = device->group->iommu_group;
560	struct vfio_iommu_driver *driver;	549	struct vfio_iommu_driver *driver = container->iommu_driver;
561	int ret;
562
563	if (!pages \|\| !npage \|\| !vfio_assert_device_open(device))
564	return -EINVAL;
565		550
566	if (npage > VFIO_PIN_PAGES_MAX_ENTRIES)	551	if (npage > VFIO_PIN_PAGES_MAX_ENTRIES)
567	return -E2BIG;	552	return -E2BIG;
568		553
569	/* group->container cannot change while a vfio device is open */	554	if (unlikely(!driver \|\| !driver->ops->pin_pages))
570	container = group->container;	555	return -ENOTTY;
571	driver = container->iommu_driver;	556	return driver->ops->pin_pages(container->iommu_data, iommu_group, iova,
572	if (likely(driver && driver->ops->pin_pages))	557	npage, prot, pages);
573	ret = driver->ops->pin_pages(container->iommu_data,
574	group->iommu_group, iova,
575	npage, prot, pages);
576	else
577	ret = -ENOTTY;
578
579	return ret;
580	}	558	}
581	EXPORT_SYMBOL(vfio_pin_pages);
582		559
583	/*	560	void vfio_device_container_unpin_pages(struct vfio_device *device,
584	* Unpin contiguous host pages for local domain only.	561	dma_addr_t iova, int npage)
585	* @device [in] : device
586	* @iova [in] : starting address of user pages to be unpinned.
587	* @npage [in] : count of pages to be unpinned. This count should not
588	* be greater than VFIO_PIN_PAGES_MAX_ENTRIES.
589	*/
590	void vfio_unpin_pages(struct vfio_device *device, dma_addr_t iova, int npage)
591	{	562	{
592	struct vfio_container *container;	563	struct vfio_container *container = device->group->container;
593	struct vfio_iommu_driver *driver;
594		564
595	if (WARN_ON(npage <= 0 \|\| npage > VFIO_PIN_PAGES_MAX_ENTRIES))	565	if (WARN_ON(npage <= 0 \|\| npage > VFIO_PIN_PAGES_MAX_ENTRIES))
596	return;	566	return;
597		567
598	if (WARN_ON(!vfio_assert_device_open(device)))	568	container->iommu_driver->ops->unpin_pages(container->iommu_data, iova,
599	return;	569	npage);
600
601	/* group->container cannot change while a vfio device is open */
602	container = device->group->container;
603	driver = container->iommu_driver;
604
605	driver->ops->unpin_pages(container->iommu_data, iova, npage);
606	}	570	}
607	EXPORT_SYMBOL(vfio_unpin_pages);
608		571
609	/*	572	int vfio_device_container_dma_rw(struct vfio_device *device,
610	* This interface allows the CPUs to perform some sort of virtual DMA on	573	dma_addr_t iova, void *data,
611	* behalf of the device.	574	size_t len, bool write)
612	*
613	* CPUs read/write from/into a range of IOVAs pointing to user space memory
614	* into/from a kernel buffer.
615	*
616	* As the read/write of user space memory is conducted via the CPUs and is
617	* not a real device DMA, it is not necessary to pin the user space memory.
618	*
619	* @device [in] : VFIO device
620	* @iova [in] : base IOVA of a user space buffer
621	* @data [in] : pointer to kernel buffer
622	* @len [in] : kernel buffer length
623	* @write : indicate read or write
624	* Return error code on failure or 0 on success.
625	*/
626	int vfio_dma_rw(struct vfio_device device, dma_addr_t iova, void data,
627	size_t len, bool write)
628	{	575	{
629	struct vfio_container *container;	576	struct vfio_container *container = device->group->container;
630	struct vfio_iommu_driver *driver;	577	struct vfio_iommu_driver *driver = container->iommu_driver;
631	int ret = 0;
632
633	if (!data \|\| len <= 0 \|\| !vfio_assert_device_open(device))
634	return -EINVAL;
635
636	/* group->container cannot change while a vfio device is open */
637	container = device->group->container;
638	driver = container->iommu_driver;
639		578
640	if (likely(driver && driver->ops->dma_rw))	579	if (unlikely(!driver \|\| !driver->ops->dma_rw))
641	ret = driver->ops->dma_rw(container->iommu_data,	580	return -ENOTTY;
642	iova, data, len, write);	581	return driver->ops->dma_rw(container->iommu_data, iova, data, len,
643	else	582	write);
644	ret = -ENOTTY;
645	return ret;
646	}	583	}
647	EXPORT_SYMBOL(vfio_dma_rw);
648		584
649	int __init vfio_container_init(void)	585	int __init vfio_container_init(void)
650	{	586	{
@@ -678,3 +614,6 @@ void vfio_container_cleanup(void) · 27 unmodified lines
678	misc_deregister(&vfio_dev);	614	misc_deregister(&vfio_dev);
679	mutex_destroy(&vfio.iommu_drivers_lock);	615	mutex_destroy(&vfio.iommu_drivers_lock);
680	}	616	}
		617
		618	MODULE_ALIAS_MISCDEV(VFIO_MINOR);
		619	MODULE_ALIAS("devname:vfio/vfio");

drivers/vfio/fsl-mc/vfio_fsl_mc.c+3−0

 	.read		= vfio_fsl_mc_read,
 	.write		= vfio_fsl_mc_write,
 	.mmap		= vfio_fsl_mc_mmap,
+	.bind_iommufd	= vfio_iommufd_physical_bind,
+	.unbind_iommufd	= vfio_iommufd_physical_unbind,
+	.attach_ioas	= vfio_iommufd_physical_attach_ioas,
 };
 static struct fsl_mc_driver vfio_fsl_mc_driver = {

drivers/vfio/group.c+877−0

@@ -0,0 +1,877 @@
	1	// SPDX-License-Identifier: GPL-2.0-only
	2	/*
	3	* VFIO core
	4	*
	5	* Copyright (C) 2012 Red Hat, Inc. All rights reserved.
	6	* Author: Alex Williamson <alex.williamson@redhat.com>
	7	*
	8	* Derived from original vfio:
	9	* Copyright 2010 Cisco Systems, Inc. All rights reserved.
	10	* Author: Tom Lyon, pugs@cisco.com
	11	*/
	12
	13	#include <linux/vfio.h>
	14	#include <linux/iommufd.h>
	15	#include <linux/anon_inodes.h>
	16	#include "vfio.h"
	17
	18	static struct vfio {
	19	struct class *class;
	20	struct list_head group_list;
	21	struct mutex group_lock; /* locks group_list */
	22	struct ida group_ida;
	23	dev_t group_devt;
	24	} vfio;
	25
	26	static struct vfio_device vfio_device_get_from_name(struct vfio_group group,
	27	char *buf)
	28	{
	29	struct vfio_device it, device = ERR_PTR(-ENODEV);
	30
	31	mutex_lock(&group->device_lock);
	32	list_for_each_entry(it, &group->device_list, group_next) {
	33	int ret;
	34
	35	if (it->ops->match) {
	36	ret = it->ops->match(it, buf);
	37	if (ret < 0) {
	38	device = ERR_PTR(ret);
	39	break;
	40	}
	41	} else {
	42	ret = !strcmp(dev_name(it->dev), buf);
	43	}
	44
	45	if (ret && vfio_device_try_get_registration(it)) {
	46	device = it;
	47	break;
	48	}
	49	}
	50	mutex_unlock(&group->device_lock);
	51
	52	return device;
	53	}
	54
	55	/*
	56	* VFIO Group fd, /dev/vfio/$GROUP
	57	*/
	58	static bool vfio_group_has_iommu(struct vfio_group *group)
	59	{
	60	lockdep_assert_held(&group->group_lock);
	61	/*
	62	* There can only be users if there is a container, and if there is a
	63	* container there must be users.
	64	*/
	65	WARN_ON(!group->container != !group->container_users);
	66
	67	return group->container \|\| group->iommufd;
	68	}
	69
	70	/*
	71	* VFIO_GROUP_UNSET_CONTAINER should fail if there are other users or
	72	* if there was no container to unset. Since the ioctl is called on
	73	* the group, we know that still exists, therefore the only valid
	74	* transition here is 1->0.
	75	*/
	76	static int vfio_group_ioctl_unset_container(struct vfio_group *group)
	77	{
	78	int ret = 0;
	79
	80	mutex_lock(&group->group_lock);
	81	if (!vfio_group_has_iommu(group)) {
	82	ret = -EINVAL;
	83	goto out_unlock;
	84	}
	85	if (group->container) {
	86	if (group->container_users != 1) {
	87	ret = -EBUSY;
	88	goto out_unlock;
	89	}
	90	vfio_group_detach_container(group);
	91	}
	92	if (group->iommufd) {
	93	iommufd_ctx_put(group->iommufd);
	94	group->iommufd = NULL;
	95	}
	96
	97	out_unlock:
	98	mutex_unlock(&group->group_lock);
	99	return ret;
	100	}
	101
	102	static int vfio_group_ioctl_set_container(struct vfio_group *group,
	103	int __user *arg)
	104	{
	105	struct vfio_container *container;
	106	struct iommufd_ctx *iommufd;
	107	struct fd f;
	108	int ret;
	109	int fd;
	110
	111	if (get_user(fd, arg))
	112	return -EFAULT;
	113
	114	f = fdget(fd);
	115	if (!f.file)
	116	return -EBADF;
	117
	118	mutex_lock(&group->group_lock);
	119	if (vfio_group_has_iommu(group)) {
	120	ret = -EINVAL;
	121	goto out_unlock;
	122	}
	123	if (!group->iommu_group) {
	124	ret = -ENODEV;
	125	goto out_unlock;
	126	}
	127
	128	container = vfio_container_from_file(f.file);
	129	if (container) {
	130	ret = vfio_container_attach_group(container, group);
	131	goto out_unlock;
	132	}
	133
	134	iommufd = iommufd_ctx_from_file(f.file);
	135	if (!IS_ERR(iommufd)) {
	136	u32 ioas_id;
	137
	138	ret = iommufd_vfio_compat_ioas_id(iommufd, &ioas_id);
	139	if (ret) {
	140	iommufd_ctx_put(group->iommufd);
	141	goto out_unlock;
	142	}
	143
	144	group->iommufd = iommufd;
	145	goto out_unlock;
	146	}
	147
	148	/* The FD passed is not recognized. */
	149	ret = -EBADFD;
	150
	151	out_unlock:
	152	mutex_unlock(&group->group_lock);
	153	fdput(f);
	154	return ret;
	155	}
	156
	157	static int vfio_device_group_open(struct vfio_device *device)
	158	{
	159	int ret;
	160
	161	mutex_lock(&device->group->group_lock);
	162	if (!vfio_group_has_iommu(device->group)) {
	163	ret = -EINVAL;
	164	goto out_unlock;
	165	}
	166
	167	/*
	168	* Here we pass the KVM pointer with the group under the lock. If the
	169	* device driver will use it, it must obtain a reference and release it
	170	* during close_device.
	171	*/
	172	ret = vfio_device_open(device, device->group->iommufd,
	173	device->group->kvm);
	174
	175	out_unlock:
	176	mutex_unlock(&device->group->group_lock);
	177	return ret;
	178	}
	179
	180	void vfio_device_group_close(struct vfio_device *device)
	181	{
	182	mutex_lock(&device->group->group_lock);
	183	vfio_device_close(device, device->group->iommufd);
	184	mutex_unlock(&device->group->group_lock);
	185	}
	186
	187	static struct file vfio_device_open_file(struct vfio_device device)
	188	{
	189	struct file *filep;
	190	int ret;
	191
	192	ret = vfio_device_group_open(device);
	193	if (ret)
	194	goto err_out;
	195
	196	/*
	197	* We can't use anon_inode_getfd() because we need to modify
	198	* the f_mode flags directly to allow more than just ioctls
	199	*/
	200	filep = anon_inode_getfile("[vfio-device]", &vfio_device_fops,
	201	device, O_RDWR);
	202	if (IS_ERR(filep)) {
	203	ret = PTR_ERR(filep);
	204	goto err_close_device;
	205	}
	206
	207	/*
	208	* TODO: add an anon_inode interface to do this.
	209	* Appears to be missing by lack of need rather than
	210	* explicitly prevented. Now there's need.
	211	*/
	212	filep->f_mode \|= (FMODE_PREAD \| FMODE_PWRITE);
	213
	214	if (device->group->type == VFIO_NO_IOMMU)
	215	dev_warn(device->dev, "vfio-noiommu device opened by user "
	216	"(%s:%d)\n", current->comm, task_pid_nr(current));
	217	/*
	218	* On success the ref of device is moved to the file and
	219	* put in vfio_device_fops_release()
	220	*/
	221	return filep;
	222
	223	err_close_device:
	224	vfio_device_group_close(device);
	225	err_out:
	226	return ERR_PTR(ret);
	227	}
	228
	229	static int vfio_group_ioctl_get_device_fd(struct vfio_group *group,
	230	char __user *arg)
	231	{
	232	struct vfio_device *device;
	233	struct file *filep;
	234	char *buf;
	235	int fdno;
	236	int ret;
	237
	238	buf = strndup_user(arg, PAGE_SIZE);
	239	if (IS_ERR(buf))
	240	return PTR_ERR(buf);
	241
	242	device = vfio_device_get_from_name(group, buf);
	243	kfree(buf);
	244	if (IS_ERR(device))
	245	return PTR_ERR(device);
	246
	247	fdno = get_unused_fd_flags(O_CLOEXEC);
	248	if (fdno < 0) {
	249	ret = fdno;
	250	goto err_put_device;
	251	}
	252
	253	filep = vfio_device_open_file(device);
	254	if (IS_ERR(filep)) {
	255	ret = PTR_ERR(filep);
	256	goto err_put_fdno;
	257	}
	258
	259	fd_install(fdno, filep);
	260	return fdno;
	261
	262	err_put_fdno:
	263	put_unused_fd(fdno);
	264	err_put_device:
	265	vfio_device_put_registration(device);
	266	return ret;
	267	}
	268
	269	static int vfio_group_ioctl_get_status(struct vfio_group *group,
	270	struct vfio_group_status __user *arg)
	271	{
	272	unsigned long minsz = offsetofend(struct vfio_group_status, flags);
	273	struct vfio_group_status status;
	274
	275	if (copy_from_user(&status, arg, minsz))
	276	return -EFAULT;
	277
	278	if (status.argsz < minsz)
	279	return -EINVAL;
	280
	281	status.flags = 0;
	282
	283	mutex_lock(&group->group_lock);
	284	if (!group->iommu_group) {
	285	mutex_unlock(&group->group_lock);
	286	return -ENODEV;
	287	}
	288
	289	/*
	290	* With the container FD the iommu_group_claim_dma_owner() is done
	291	* during SET_CONTAINER but for IOMMFD this is done during
	292	* VFIO_GROUP_GET_DEVICE_FD. Meaning that with iommufd
	293	* VFIO_GROUP_FLAGS_VIABLE could be set but GET_DEVICE_FD will fail due
	294	* to viability.
	295	*/
	296	if (vfio_group_has_iommu(group))
	297	status.flags \|= VFIO_GROUP_FLAGS_CONTAINER_SET \|
	298	VFIO_GROUP_FLAGS_VIABLE;
	299	else if (!iommu_group_dma_owner_claimed(group->iommu_group))
	300	status.flags \|= VFIO_GROUP_FLAGS_VIABLE;
	301	mutex_unlock(&group->group_lock);
	302
	303	if (copy_to_user(arg, &status, minsz))
	304	return -EFAULT;
	305	return 0;
	306	}
	307
	308	static long vfio_group_fops_unl_ioctl(struct file *filep,
	309	unsigned int cmd, unsigned long arg)
	310	{
	311	struct vfio_group *group = filep->private_data;
	312	void __user uarg = (void __user )arg;
	313
	314	switch (cmd) {
	315	case VFIO_GROUP_GET_DEVICE_FD:
	316	return vfio_group_ioctl_get_device_fd(group, uarg);
	317	case VFIO_GROUP_GET_STATUS:
	318	return vfio_group_ioctl_get_status(group, uarg);
	319	case VFIO_GROUP_SET_CONTAINER:
	320	return vfio_group_ioctl_set_container(group, uarg);
	321	case VFIO_GROUP_UNSET_CONTAINER:
	322	return vfio_group_ioctl_unset_container(group);
	323	default:
	324	return -ENOTTY;
	325	}
	326	}
	327
	328	static int vfio_group_fops_open(struct inode inode, struct file filep)
	329	{
	330	struct vfio_group *group =
	331	container_of(inode->i_cdev, struct vfio_group, cdev);
	332	int ret;
	333
	334	mutex_lock(&group->group_lock);
	335
	336	/*
	337	* drivers can be zero if this races with vfio_device_remove_group(), it
	338	* will be stable at 0 under the group rwsem
	339	*/
	340	if (refcount_read(&group->drivers) == 0) {
	341	ret = -ENODEV;
	342	goto out_unlock;
	343	}
	344
	345	if (group->type == VFIO_NO_IOMMU && !capable(CAP_SYS_RAWIO)) {
	346	ret = -EPERM;
	347	goto out_unlock;
	348	}
	349
	350	/*
	351	* Do we need multiple instances of the group open? Seems not.
	352	*/
	353	if (group->opened_file) {
	354	ret = -EBUSY;
	355	goto out_unlock;
	356	}
	357	group->opened_file = filep;
	358	filep->private_data = group;
	359	ret = 0;
	360	out_unlock:
	361	mutex_unlock(&group->group_lock);
	362	return ret;
	363	}
	364
	365	static int vfio_group_fops_release(struct inode inode, struct file filep)
	366	{
	367	struct vfio_group *group = filep->private_data;
	368
	369	filep->private_data = NULL;
	370
	371	mutex_lock(&group->group_lock);
	372	/*
	373	* Device FDs hold a group file reference, therefore the group release
	374	* is only called when there are no open devices.
	375	*/
	376	WARN_ON(group->notifier.head);
	377	if (group->container)
	378	vfio_group_detach_container(group);
	379	if (group->iommufd) {
	380	iommufd_ctx_put(group->iommufd);
	381	group->iommufd = NULL;
	382	}
	383	group->opened_file = NULL;
	384	mutex_unlock(&group->group_lock);
	385	return 0;
	386	}
	387
	388	static const struct file_operations vfio_group_fops = {
	389	.owner = THIS_MODULE,
	390	.unlocked_ioctl = vfio_group_fops_unl_ioctl,
	391	.compat_ioctl = compat_ptr_ioctl,
	392	.open = vfio_group_fops_open,
	393	.release = vfio_group_fops_release,
	394	};
	395
	396	/*
	397	* Group objects - create, release, get, put, search
	398	*/
	399	static struct vfio_group *
	400	vfio_group_find_from_iommu(struct iommu_group *iommu_group)
	401	{
	402	struct vfio_group *group;
	403
	404	lockdep_assert_held(&vfio.group_lock);
	405
	406	/*
	407	* group->iommu_group from the vfio.group_list cannot be NULL
	408	* under the vfio.group_lock.
	409	*/
	410	list_for_each_entry(group, &vfio.group_list, vfio_next) {
	411	if (group->iommu_group == iommu_group)
	412	return group;
	413	}
	414	return NULL;
	415	}
	416
	417	static void vfio_group_release(struct device *dev)
	418	{
	419	struct vfio_group *group = container_of(dev, struct vfio_group, dev);
	420
	421	mutex_destroy(&group->device_lock);
	422	mutex_destroy(&group->group_lock);
	423	WARN_ON(group->iommu_group);
	424	ida_free(&vfio.group_ida, MINOR(group->dev.devt));
	425	kfree(group);
	426	}
	427
	428	static struct vfio_group vfio_group_alloc(struct iommu_group iommu_group,
	429	enum vfio_group_type type)
	430	{
	431	struct vfio_group *group;
	432	int minor;
	433
	434	group = kzalloc(sizeof(*group), GFP_KERNEL);
	435	if (!group)
	436	return ERR_PTR(-ENOMEM);
	437
	438	minor = ida_alloc_max(&vfio.group_ida, MINORMASK, GFP_KERNEL);
	439	if (minor < 0) {
	440	kfree(group);
	441	return ERR_PTR(minor);
	442	}
	443
	444	device_initialize(&group->dev);
	445	group->dev.devt = MKDEV(MAJOR(vfio.group_devt), minor);
	446	group->dev.class = vfio.class;
	447	group->dev.release = vfio_group_release;
	448	cdev_init(&group->cdev, &vfio_group_fops);
	449	group->cdev.owner = THIS_MODULE;
	450
	451	refcount_set(&group->drivers, 1);
	452	mutex_init(&group->group_lock);
	453	INIT_LIST_HEAD(&group->device_list);
	454	mutex_init(&group->device_lock);
	455	group->iommu_group = iommu_group;
	456	/* put in vfio_group_release() */
	457	iommu_group_ref_get(iommu_group);
	458	group->type = type;
	459	BLOCKING_INIT_NOTIFIER_HEAD(&group->notifier);
	460
	461	return group;
	462	}
	463
	464	static struct vfio_group vfio_create_group(struct iommu_group iommu_group,
	465	enum vfio_group_type type)
	466	{
	467	struct vfio_group *group;
	468	struct vfio_group *ret;
	469	int err;
	470
	471	lockdep_assert_held(&vfio.group_lock);
	472
	473	group = vfio_group_alloc(iommu_group, type);
	474	if (IS_ERR(group))
	475	return group;
	476
	477	err = dev_set_name(&group->dev, "%s%d",
	478	group->type == VFIO_NO_IOMMU ? "noiommu-" : "",
	479	iommu_group_id(iommu_group));
	480	if (err) {
	481	ret = ERR_PTR(err);
	482	goto err_put;
	483	}
	484
	485	err = cdev_device_add(&group->cdev, &group->dev);
	486	if (err) {
	487	ret = ERR_PTR(err);
	488	goto err_put;
	489	}
	490
	491	list_add(&group->vfio_next, &vfio.group_list);
	492
	493	return group;
	494
	495	err_put:
	496	put_device(&group->dev);
	497	return ret;
	498	}
	499
	500	static struct vfio_group vfio_noiommu_group_alloc(struct device dev,
	501	enum vfio_group_type type)
	502	{
	503	struct iommu_group *iommu_group;
	504	struct vfio_group *group;
	505	int ret;
	506
	507	iommu_group = iommu_group_alloc();
	508	if (IS_ERR(iommu_group))
	509	return ERR_CAST(iommu_group);
	510
	511	ret = iommu_group_set_name(iommu_group, "vfio-noiommu");
	512	if (ret)
	513	goto out_put_group;
	514	ret = iommu_group_add_device(iommu_group, dev);
	515	if (ret)
	516	goto out_put_group;
	517
	518	mutex_lock(&vfio.group_lock);
	519	group = vfio_create_group(iommu_group, type);
	520	mutex_unlock(&vfio.group_lock);
	521	if (IS_ERR(group)) {
	522	ret = PTR_ERR(group);
	523	goto out_remove_device;
	524	}
	525	iommu_group_put(iommu_group);
	526	return group;
	527
	528	out_remove_device:
	529	iommu_group_remove_device(dev);
	530	out_put_group:
	531	iommu_group_put(iommu_group);
	532	return ERR_PTR(ret);
	533	}
	534
	535	static bool vfio_group_has_device(struct vfio_group group, struct device dev)
	536	{
	537	struct vfio_device *device;
	538
	539	mutex_lock(&group->device_lock);
	540	list_for_each_entry(device, &group->device_list, group_next) {
	541	if (device->dev == dev) {
	542	mutex_unlock(&group->device_lock);
	543	return true;
	544	}
	545	}
	546	mutex_unlock(&group->device_lock);
	547	return false;
	548	}
	549
	550	static struct vfio_group vfio_group_find_or_alloc(struct device dev)
	551	{
	552	struct iommu_group *iommu_group;
	553	struct vfio_group *group;
	554
	555	iommu_group = iommu_group_get(dev);
	556	if (!iommu_group && vfio_noiommu) {
	557	/*
	558	* With noiommu enabled, create an IOMMU group for devices that
	559	* don't already have one, implying no IOMMU hardware/driver
	560	* exists. Taint the kernel because we're about to give a DMA
	561	* capable device to a user without IOMMU protection.
	562	*/
	563	group = vfio_noiommu_group_alloc(dev, VFIO_NO_IOMMU);
	564	if (!IS_ERR(group)) {
	565	add_taint(TAINT_USER, LOCKDEP_STILL_OK);
	566	dev_warn(dev, "Adding kernel taint for vfio-noiommu group on device\n");
	567	}
	568	return group;
	569	}
	570
	571	if (!iommu_group)
	572	return ERR_PTR(-EINVAL);
	573
	574	/*
	575	* VFIO always sets IOMMU_CACHE because we offer no way for userspace to
	576	* restore cache coherency. It has to be checked here because it is only
	577	* valid for cases where we are using iommu groups.
	578	*/
	579	if (!device_iommu_capable(dev, IOMMU_CAP_CACHE_COHERENCY)) {
	580	iommu_group_put(iommu_group);
	581	return ERR_PTR(-EINVAL);
	582	}
	583
	584	mutex_lock(&vfio.group_lock);
	585	group = vfio_group_find_from_iommu(iommu_group);
	586	if (group) {
	587	if (WARN_ON(vfio_group_has_device(group, dev)))
	588	group = ERR_PTR(-EINVAL);
	589	else
	590	refcount_inc(&group->drivers);
	591	} else {
	592	group = vfio_create_group(iommu_group, VFIO_IOMMU);
	593	}
	594	mutex_unlock(&vfio.group_lock);
	595
	596	/* The vfio_group holds a reference to the iommu_group */
	597	iommu_group_put(iommu_group);
	598	return group;
	599	}
	600
	601	int vfio_device_set_group(struct vfio_device *device,
	602	enum vfio_group_type type)
	603	{
	604	struct vfio_group *group;
	605
	606	if (type == VFIO_IOMMU)
	607	group = vfio_group_find_or_alloc(device->dev);
	608	else
	609	group = vfio_noiommu_group_alloc(device->dev, type);
	610
	611	if (IS_ERR(group))
	612	return PTR_ERR(group);
	613
	614	/* Our reference on group is moved to the device */
	615	device->group = group;
	616	return 0;
	617	}
	618
	619	void vfio_device_remove_group(struct vfio_device *device)
	620	{
	621	struct vfio_group *group = device->group;
	622	struct iommu_group *iommu_group;
	623
	624	if (group->type == VFIO_NO_IOMMU \|\| group->type == VFIO_EMULATED_IOMMU)
	625	iommu_group_remove_device(device->dev);
	626
	627	/* Pairs with vfio_create_group() / vfio_group_get_from_iommu() */
	628	if (!refcount_dec_and_mutex_lock(&group->drivers, &vfio.group_lock))
	629	return;
	630	list_del(&group->vfio_next);
	631
	632	/*
	633	* We could concurrently probe another driver in the group that might
	634	* race vfio_device_remove_group() with vfio_get_group(), so we have to
	635	* ensure that the sysfs is all cleaned up under lock otherwise the
	636	* cdev_device_add() will fail due to the name aready existing.
	637	*/
	638	cdev_device_del(&group->cdev, &group->dev);
	639
	640	mutex_lock(&group->group_lock);
	641	/*
	642	* These data structures all have paired operations that can only be
	643	* undone when the caller holds a live reference on the device. Since
	644	* all pairs must be undone these WARN_ON's indicate some caller did not
	645	* properly hold the group reference.
	646	*/
	647	WARN_ON(!list_empty(&group->device_list));
	648	WARN_ON(group->notifier.head);
	649
	650	/*
	651	* Revoke all users of group->iommu_group. At this point we know there
	652	* are no devices active because we are unplugging the last one. Setting
	653	* iommu_group to NULL blocks all new users.
	654	*/
	655	if (group->container)
	656	vfio_group_detach_container(group);
	657	iommu_group = group->iommu_group;
	658	group->iommu_group = NULL;
	659	mutex_unlock(&group->group_lock);
	660	mutex_unlock(&vfio.group_lock);
	661
	662	iommu_group_put(iommu_group);
	663	put_device(&group->dev);
	664	}
	665
	666	void vfio_device_group_register(struct vfio_device *device)
	667	{
	668	mutex_lock(&device->group->device_lock);
	669	list_add(&device->group_next, &device->group->device_list);
	670	mutex_unlock(&device->group->device_lock);
	671	}
	672
	673	void vfio_device_group_unregister(struct vfio_device *device)
	674	{
	675	mutex_lock(&device->group->device_lock);
	676	list_del(&device->group_next);
	677	mutex_unlock(&device->group->device_lock);
	678	}
	679
	680	int vfio_device_group_use_iommu(struct vfio_device *device)
	681	{
	682	struct vfio_group *group = device->group;
	683	int ret = 0;
	684
	685	lockdep_assert_held(&group->group_lock);
	686
	687	if (WARN_ON(!group->container))
	688	return -EINVAL;
	689
	690	ret = vfio_group_use_container(group);
	691	if (ret)
	692	return ret;
	693	vfio_device_container_register(device);
	694	return 0;
	695	}
	696
	697	void vfio_device_group_unuse_iommu(struct vfio_device *device)
	698	{
	699	struct vfio_group *group = device->group;
	700
	701	lockdep_assert_held(&group->group_lock);
	702
	703	if (WARN_ON(!group->container))
	704	return;
	705
	706	vfio_device_container_unregister(device);
	707	vfio_group_unuse_container(group);
	708	}
	709
	710	bool vfio_device_has_container(struct vfio_device *device)
	711	{
	712	return device->group->container;
	713	}
	714
	715	/**
	716	* vfio_file_iommu_group - Return the struct iommu_group for the vfio group file
	717	* @file: VFIO group file
	718	*
	719	* The returned iommu_group is valid as long as a ref is held on the file. This
	720	* returns a reference on the group. This function is deprecated, only the SPAPR
	721	* path in kvm should call it.
	722	*/
	723	struct iommu_group vfio_file_iommu_group(struct file file)
	724	{
	725	struct vfio_group *group = file->private_data;
	726	struct iommu_group *iommu_group = NULL;
	727
	728	if (!IS_ENABLED(CONFIG_SPAPR_TCE_IOMMU))
	729	return NULL;
	730
	731	if (!vfio_file_is_group(file))
	732	return NULL;
	733
	734	mutex_lock(&group->group_lock);
	735	if (group->iommu_group) {
	736	iommu_group = group->iommu_group;
	737	iommu_group_ref_get(iommu_group);
	738	}
	739	mutex_unlock(&group->group_lock);
	740	return iommu_group;
	741	}
	742	EXPORT_SYMBOL_GPL(vfio_file_iommu_group);
	743
	744	/**
	745	* vfio_file_is_group - True if the file is usable with VFIO aPIS
	746	* @file: VFIO group file
	747	*/
	748	bool vfio_file_is_group(struct file *file)
	749	{
	750	return file->f_op == &vfio_group_fops;
	751	}
	752	EXPORT_SYMBOL_GPL(vfio_file_is_group);
	753
	754	/**
	755	* vfio_file_enforced_coherent - True if the DMA associated with the VFIO file
	756	* is always CPU cache coherent
	757	* @file: VFIO group file
	758	*
	759	* Enforced coherency means that the IOMMU ignores things like the PCIe no-snoop
	760	* bit in DMA transactions. A return of false indicates that the user has
	761	* rights to access additional instructions such as wbinvd on x86.
	762	*/
	763	bool vfio_file_enforced_coherent(struct file *file)
	764	{
	765	struct vfio_group *group = file->private_data;
	766	struct vfio_device *device;
	767	bool ret = true;
	768
	769	if (!vfio_file_is_group(file))
	770	return true;
	771
	772	/*
	773	* If the device does not have IOMMU_CAP_ENFORCE_CACHE_COHERENCY then
	774	* any domain later attached to it will also not support it. If the cap
	775	* is set then the iommu_domain eventually attached to the device/group
	776	* must use a domain with enforce_cache_coherency().
	777	*/
	778	mutex_lock(&group->device_lock);
	779	list_for_each_entry(device, &group->device_list, group_next) {
	780	if (!device_iommu_capable(device->dev,
	781	IOMMU_CAP_ENFORCE_CACHE_COHERENCY)) {
	782	ret = false;
	783	break;
	784	}
	785	}
	786	mutex_unlock(&group->device_lock);
	787	return ret;
	788	}
	789	EXPORT_SYMBOL_GPL(vfio_file_enforced_coherent);
	790
	791	/**
	792	* vfio_file_set_kvm - Link a kvm with VFIO drivers
	793	* @file: VFIO group file
	794	* @kvm: KVM to link
	795	*
	796	* When a VFIO device is first opened the KVM will be available in
	797	* device->kvm if one was associated with the group.
	798	*/
	799	void vfio_file_set_kvm(struct file file, struct kvm kvm)
	800	{
	801	struct vfio_group *group = file->private_data;
	802
	803	if (!vfio_file_is_group(file))
	804	return;
	805
	806	mutex_lock(&group->group_lock);
	807	group->kvm = kvm;
	808	mutex_unlock(&group->group_lock);
	809	}
	810	EXPORT_SYMBOL_GPL(vfio_file_set_kvm);
	811
	812	/**
	813	* vfio_file_has_dev - True if the VFIO file is a handle for device
	814	* @file: VFIO file to check
	815	* @device: Device that must be part of the file
	816	*
	817	* Returns true if given file has permission to manipulate the given device.
	818	*/
	819	bool vfio_file_has_dev(struct file file, struct vfio_device device)
	820	{
	821	struct vfio_group *group = file->private_data;
	822
	823	if (!vfio_file_is_group(file))
	824	return false;
	825
	826	return group == device->group;
	827	}
	828	EXPORT_SYMBOL_GPL(vfio_file_has_dev);
	829
	830	static char vfio_devnode(struct device dev, umode_t *mode)
	831	{
	832	return kasprintf(GFP_KERNEL, "vfio/%s", dev_name(dev));
	833	}
	834
	835	int __init vfio_group_init(void)
	836	{
	837	int ret;
	838
	839	ida_init(&vfio.group_ida);
	840	mutex_init(&vfio.group_lock);
	841	INIT_LIST_HEAD(&vfio.group_list);
	842
	843	ret = vfio_container_init();
	844	if (ret)
	845	return ret;
	846
	847	/* /dev/vfio/$GROUP */
	848	vfio.class = class_create(THIS_MODULE, "vfio");
	849	if (IS_ERR(vfio.class)) {
	850	ret = PTR_ERR(vfio.class);
	851	goto err_group_class;
	852	}
	853
	854	vfio.class->devnode = vfio_devnode;
	855
	856	ret = alloc_chrdev_region(&vfio.group_devt, 0, MINORMASK + 1, "vfio");
	857	if (ret)
	858	goto err_alloc_chrdev;
	859	return 0;
	860
	861	err_alloc_chrdev:
	862	class_destroy(vfio.class);
	863	vfio.class = NULL;
	864	err_group_class:
	865	vfio_container_cleanup();
	866	return ret;
	867	}
	868
	869	void vfio_group_cleanup(void)
	870	{
	871	WARN_ON(!list_empty(&vfio.group_list));
	872	ida_destroy(&vfio.group_ida);
	873	unregister_chrdev_region(vfio.group_devt, MINORMASK + 1);
	874	class_destroy(vfio.class);
	875	vfio.class = NULL;
	876	vfio_container_cleanup();
	877	}

drivers/vfio/iommufd.c+158−0

@@ -0,0 +1,158 @@
	1	// SPDX-License-Identifier: GPL-2.0-only
	2	/*
	3	* Copyright (c) 2021-2022, NVIDIA CORPORATION & AFFILIATES
	4	*/
	5	#include <linux/vfio.h>
	6	#include <linux/iommufd.h>
	7
	8	#include "vfio.h"
	9
	10	MODULE_IMPORT_NS(IOMMUFD);
	11	MODULE_IMPORT_NS(IOMMUFD_VFIO);
	12
	13	int vfio_iommufd_bind(struct vfio_device vdev, struct iommufd_ctx ictx)
	14	{
	15	u32 ioas_id;
	16	u32 device_id;
	17	int ret;
	18
	19	lockdep_assert_held(&vdev->dev_set->lock);
	20
	21	/*
	22	* If the driver doesn't provide this op then it means the device does
	23	* not do DMA at all. So nothing to do.
	24	*/
	25	if (!vdev->ops->bind_iommufd)
	26	return 0;
	27
	28	ret = vdev->ops->bind_iommufd(vdev, ictx, &device_id);
	29	if (ret)
	30	return ret;
	31
	32	ret = iommufd_vfio_compat_ioas_id(ictx, &ioas_id);
	33	if (ret)
	34	goto err_unbind;
	35	ret = vdev->ops->attach_ioas(vdev, &ioas_id);
	36	if (ret)
	37	goto err_unbind;
	38
	39	/*
	40	* The legacy path has no way to return the device id or the selected
	41	* pt_id
	42	*/
	43	return 0;
	44
	45	err_unbind:
	46	if (vdev->ops->unbind_iommufd)
	47	vdev->ops->unbind_iommufd(vdev);
	48	return ret;
	49	}
	50
	51	void vfio_iommufd_unbind(struct vfio_device *vdev)
	52	{
	53	lockdep_assert_held(&vdev->dev_set->lock);
	54
	55	if (vdev->ops->unbind_iommufd)
	56	vdev->ops->unbind_iommufd(vdev);
	57	}
	58
	59	/*
	60	* The physical standard ops mean that the iommufd_device is bound to the
	61	* physical device vdev->dev that was provided to vfio_init_group_dev(). Drivers
	62	* using this ops set should call vfio_register_group_dev()
	63	*/
	64	int vfio_iommufd_physical_bind(struct vfio_device *vdev,
	65	struct iommufd_ctx ictx, u32 out_device_id)
	66	{
	67	struct iommufd_device *idev;
	68
	69	idev = iommufd_device_bind(ictx, vdev->dev, out_device_id);
	70	if (IS_ERR(idev))
	71	return PTR_ERR(idev);
	72	vdev->iommufd_device = idev;
	73	return 0;
	74	}
	75	EXPORT_SYMBOL_GPL(vfio_iommufd_physical_bind);
	76
	77	void vfio_iommufd_physical_unbind(struct vfio_device *vdev)
	78	{
	79	lockdep_assert_held(&vdev->dev_set->lock);
	80
	81	if (vdev->iommufd_attached) {
	82	iommufd_device_detach(vdev->iommufd_device);
	83	vdev->iommufd_attached = false;
	84	}
	85	iommufd_device_unbind(vdev->iommufd_device);
	86	vdev->iommufd_device = NULL;
	87	}
	88	EXPORT_SYMBOL_GPL(vfio_iommufd_physical_unbind);
	89
	90	int vfio_iommufd_physical_attach_ioas(struct vfio_device vdev, u32 pt_id)
	91	{
	92	int rc;
	93
	94	rc = iommufd_device_attach(vdev->iommufd_device, pt_id);
	95	if (rc)
	96	return rc;
	97	vdev->iommufd_attached = true;
	98	return 0;
	99	}
	100	EXPORT_SYMBOL_GPL(vfio_iommufd_physical_attach_ioas);
	101
	102	/*
	103	* The emulated standard ops mean that vfio_device is going to use the
	104	* "mdev path" and will call vfio_pin_pages()/vfio_dma_rw(). Drivers using this
	105	* ops set should call vfio_register_emulated_iommu_dev().
	106	*/
	107
	108	static void vfio_emulated_unmap(void *data, unsigned long iova,
	109	unsigned long length)
	110	{
	111	struct vfio_device *vdev = data;
	112
	113	vdev->ops->dma_unmap(vdev, iova, length);
	114	}
	115
	116	static const struct iommufd_access_ops vfio_user_ops = {
	117	.needs_pin_pages = 1,
	118	.unmap = vfio_emulated_unmap,
	119	};
	120
	121	int vfio_iommufd_emulated_bind(struct vfio_device *vdev,
	122	struct iommufd_ctx ictx, u32 out_device_id)
	123	{
	124	lockdep_assert_held(&vdev->dev_set->lock);
	125
	126	vdev->iommufd_ictx = ictx;
	127	iommufd_ctx_get(ictx);
	128	return 0;
	129	}
	130	EXPORT_SYMBOL_GPL(vfio_iommufd_emulated_bind);
	131
	132	void vfio_iommufd_emulated_unbind(struct vfio_device *vdev)
	133	{
	134	lockdep_assert_held(&vdev->dev_set->lock);
	135
	136	if (vdev->iommufd_access) {
	137	iommufd_access_destroy(vdev->iommufd_access);
	138	vdev->iommufd_access = NULL;
	139	}
	140	iommufd_ctx_put(vdev->iommufd_ictx);
	141	vdev->iommufd_ictx = NULL;
	142	}
	143	EXPORT_SYMBOL_GPL(vfio_iommufd_emulated_unbind);
	144
	145	int vfio_iommufd_emulated_attach_ioas(struct vfio_device vdev, u32 pt_id)
	146	{
	147	struct iommufd_access *user;
	148
	149	lockdep_assert_held(&vdev->dev_set->lock);
	150
	151	user = iommufd_access_create(vdev->iommufd_ictx, *pt_id, &vfio_user_ops,
	152	vdev);
	153	if (IS_ERR(user))
	154	return PTR_ERR(user);
	155	vdev->iommufd_access = user;
	156	return 0;
	157	}
	158	EXPORT_SYMBOL_GPL(vfio_iommufd_emulated_attach_ioas);

drivers/vfio/pci/hisilicon/hisi_acc_vfio_pci.c+6−0

 	.mmap = hisi_acc_vfio_pci_mmap,
 	.request = vfio_pci_core_request,
 	.match = vfio_pci_core_match,
+	.bind_iommufd = vfio_iommufd_physical_bind,
+	.unbind_iommufd = vfio_iommufd_physical_unbind,
+	.attach_ioas = vfio_iommufd_physical_attach_ioas,
 };
 static const struct vfio_device_ops hisi_acc_vfio_pci_ops = {
 	.mmap = vfio_pci_core_mmap,
 	.request = vfio_pci_core_request,
 	.match = vfio_pci_core_match,
+	.bind_iommufd = vfio_iommufd_physical_bind,
+	.unbind_iommufd = vfio_iommufd_physical_unbind,
+	.attach_ioas = vfio_iommufd_physical_attach_ioas,
 };
 static int hisi_acc_vfio_pci_probe(struct pci_dev *pdev, const struct pci_device_id *id)

drivers/vfio/pci/mlx5/main.c+3−0

 	.mmap = vfio_pci_core_mmap,
 	.request = vfio_pci_core_request,
 	.match = vfio_pci_core_match,
+	.bind_iommufd = vfio_iommufd_physical_bind,
+	.unbind_iommufd = vfio_iommufd_physical_unbind,
+	.attach_ioas = vfio_iommufd_physical_attach_ioas,
 };
 static int mlx5vf_pci_probe(struct pci_dev *pdev,

drivers/vfio/pci/vfio_pci.c+3−0

 	.mmap		= vfio_pci_core_mmap,
 	.request	= vfio_pci_core_request,
 	.match		= vfio_pci_core_match,
+	.bind_iommufd	= vfio_iommufd_physical_bind,
+	.unbind_iommufd	= vfio_iommufd_physical_unbind,
+	.attach_ioas	= vfio_iommufd_physical_attach_ioas,
 };
 static int vfio_pci_probe(struct pci_dev *pdev, const struct pci_device_id *id)

drivers/vfio/platform/vfio_amba.c+3−0

 	.read		= vfio_platform_read,
 	.write		= vfio_platform_write,
 	.mmap		= vfio_platform_mmap,
+	.bind_iommufd	= vfio_iommufd_physical_bind,
+	.unbind_iommufd	= vfio_iommufd_physical_unbind,
+	.attach_ioas	= vfio_iommufd_physical_attach_ioas,
 };
 static const struct amba_id pl330_ids[] = {

drivers/vfio/platform/vfio_platform.c+3−0

 	.read		= vfio_platform_read,
 	.write		= vfio_platform_write,
 	.mmap		= vfio_platform_mmap,
+	.bind_iommufd	= vfio_iommufd_physical_bind,
+	.unbind_iommufd	= vfio_iommufd_physical_unbind,
+	.attach_ioas	= vfio_iommufd_physical_attach_ioas,
 };
 static struct platform_driver vfio_platform_driver = {

drivers/vfio/vfio.h+114−6

@@ -6,14 +6,25 @@ · 5 unmodified lines
6	#ifndef __VFIO_VFIO_H__	6	#ifndef __VFIO_VFIO_H__
7	#define __VFIO_VFIO_H__	7	#define __VFIO_VFIO_H__
8		8
		9	#include <linux/file.h>
9	#include <linux/device.h>	10	#include <linux/device.h>
10	#include <linux/cdev.h>	11	#include <linux/cdev.h>
11	#include <linux/module.h>	12	#include <linux/module.h>
12		13
		14	struct iommufd_ctx;
13	struct iommu_group;	15	struct iommu_group;
14	struct vfio_device;	16	struct vfio_device;
15	struct vfio_container;	17	struct vfio_container;
16		18
		19	void vfio_device_put_registration(struct vfio_device *device);
		20	bool vfio_device_try_get_registration(struct vfio_device *device);
		21	int vfio_device_open(struct vfio_device *device,
		22	struct iommufd_ctx iommufd, struct kvm kvm);
		23	void vfio_device_close(struct vfio_device *device,
		24	struct iommufd_ctx *iommufd);
		25
		26	extern const struct file_operations vfio_device_fops;
		27
17	enum vfio_group_type {	28	enum vfio_group_type {
18	/*	29	/*
19	* Physical device with IOMMU backing.	30	* Physical device with IOMMU backing.
@@ -54,14 +65,30 @@ struct vfio_group { · 34 unmodified lines
54	struct list_head device_list;	65	struct list_head device_list;
55	struct mutex device_lock;	66	struct mutex device_lock;
56	struct list_head vfio_next;	67	struct list_head vfio_next;
		68	#if IS_ENABLED(CONFIG_VFIO_CONTAINER)
57	struct list_head container_next;	69	struct list_head container_next;
		70	#endif
58	enum vfio_group_type type;	71	enum vfio_group_type type;
59	struct mutex group_lock;	72	struct mutex group_lock;
60	struct kvm *kvm;	73	struct kvm *kvm;
61	struct file *opened_file;	74	struct file *opened_file;
62	struct blocking_notifier_head notifier;	75	struct blocking_notifier_head notifier;
		76	struct iommufd_ctx *iommufd;
63	};	77	};
64		78
		79	int vfio_device_set_group(struct vfio_device *device,
		80	enum vfio_group_type type);
		81	void vfio_device_remove_group(struct vfio_device *device);
		82	void vfio_device_group_register(struct vfio_device *device);
		83	void vfio_device_group_unregister(struct vfio_device *device);
		84	int vfio_device_group_use_iommu(struct vfio_device *device);
		85	void vfio_device_group_unuse_iommu(struct vfio_device *device);
		86	void vfio_device_group_close(struct vfio_device *device);
		87	bool vfio_device_has_container(struct vfio_device *device);
		88	int __init vfio_group_init(void);
		89	void vfio_group_cleanup(void);
		90
		91	#if IS_ENABLED(CONFIG_VFIO_CONTAINER)
65	/* events for the backend driver notify callback */	92	/* events for the backend driver notify callback */
66	enum vfio_iommu_notify_type {	93	enum vfio_iommu_notify_type {
67	VFIO_IOMMU_CONTAINER_CLOSE = 0,	94	VFIO_IOMMU_CONTAINER_CLOSE = 0,
@@ -109,20 +136,101 @@ struct vfio_iommu_driver { · 41 unmodified lines
109	int vfio_register_iommu_driver(const struct vfio_iommu_driver_ops *ops);	136	int vfio_register_iommu_driver(const struct vfio_iommu_driver_ops *ops);
110	void vfio_unregister_iommu_driver(const struct vfio_iommu_driver_ops *ops);	137	void vfio_unregister_iommu_driver(const struct vfio_iommu_driver_ops *ops);
111		138
112	bool vfio_assert_device_open(struct vfio_device *device);
113
114	struct vfio_container vfio_container_from_file(struct file filep);	139	struct vfio_container vfio_container_from_file(struct file filep);
115	int vfio_device_assign_container(struct vfio_device *device);	140	int vfio_group_use_container(struct vfio_group *group);
116	void vfio_device_unassign_container(struct vfio_device *device);	141	void vfio_group_unuse_container(struct vfio_group *group);
117	int vfio_container_attach_group(struct vfio_container *container,	142	int vfio_container_attach_group(struct vfio_container *container,
118	struct vfio_group *group);	143	struct vfio_group *group);
119	void vfio_group_detach_container(struct vfio_group *group);	144	void vfio_group_detach_container(struct vfio_group *group);
120	void vfio_device_container_register(struct vfio_device *device);	145	void vfio_device_container_register(struct vfio_device *device);
121	void vfio_device_container_unregister(struct vfio_device *device);	146	void vfio_device_container_unregister(struct vfio_device *device);
122	long vfio_container_ioctl_check_extension(struct vfio_container *container,	147	int vfio_device_container_pin_pages(struct vfio_device *device,
123	unsigned long arg);	148	dma_addr_t iova, int npage,
		149	int prot, struct page **pages);
		150	void vfio_device_container_unpin_pages(struct vfio_device *device,
		151	dma_addr_t iova, int npage);
		152	int vfio_device_container_dma_rw(struct vfio_device *device,
		153	dma_addr_t iova, void *data,
		154	size_t len, bool write);
		155
124	int __init vfio_container_init(void);	156	int __init vfio_container_init(void);
125	void vfio_container_cleanup(void);	157	void vfio_container_cleanup(void);
		158	#else
		159	static inline struct vfio_container *
		160	vfio_container_from_file(struct file *filep)
		161	{
		162	return NULL;
		163	}
		164
		165	static inline int vfio_group_use_container(struct vfio_group *group)
		166	{
		167	return -EOPNOTSUPP;
		168	}
		169
		170	static inline void vfio_group_unuse_container(struct vfio_group *group)
		171	{
		172	}
		173
		174	static inline int vfio_container_attach_group(struct vfio_container *container,
		175	struct vfio_group *group)
		176	{
		177	return -EOPNOTSUPP;
		178	}
		179
		180	static inline void vfio_group_detach_container(struct vfio_group *group)
		181	{
		182	}
		183
		184	static inline void vfio_device_container_register(struct vfio_device *device)
		185	{
		186	}
		187
		188	static inline void vfio_device_container_unregister(struct vfio_device *device)
		189	{
		190	}
		191
		192	static inline int vfio_device_container_pin_pages(struct vfio_device *device,
		193	dma_addr_t iova, int npage,
		194	int prot, struct page **pages)
		195	{
		196	return -EOPNOTSUPP;
		197	}
		198
		199	static inline void vfio_device_container_unpin_pages(struct vfio_device *device,
		200	dma_addr_t iova, int npage)
		201	{
		202	}
		203
		204	static inline int vfio_device_container_dma_rw(struct vfio_device *device,
		205	dma_addr_t iova, void *data,
		206	size_t len, bool write)
		207	{
		208	return -EOPNOTSUPP;
		209	}
		210
		211	static inline int vfio_container_init(void)
		212	{
		213	return 0;
		214	}
		215	static inline void vfio_container_cleanup(void)
		216	{
		217	}
		218	#endif
		219
		220	#if IS_ENABLED(CONFIG_IOMMUFD)
		221	int vfio_iommufd_bind(struct vfio_device device, struct iommufd_ctx ictx);
		222	void vfio_iommufd_unbind(struct vfio_device *device);
		223	#else
		224	static inline int vfio_iommufd_bind(struct vfio_device *device,
		225	struct iommufd_ctx *ictx)
		226	{
		227	return -EOPNOTSUPP;
		228	}
		229
		230	static inline void vfio_iommufd_unbind(struct vfio_device *device)
		231	{
		232	}
		233	#endif
126		234
127	#ifdef CONFIG_VFIO_NOIOMMU	235	#ifdef CONFIG_VFIO_NOIOMMU
128	extern bool vfio_noiommu __read_mostly;	236	extern bool vfio_noiommu __read_mostly;

drivers/vfio/vfio_main.c+193−790

@@ -13,8 +13,6 @@ · 12 unmodified lines
13	#include <linux/cdev.h>	13	#include <linux/cdev.h>
14	#include <linux/compat.h>	14	#include <linux/compat.h>
15	#include <linux/device.h>	15	#include <linux/device.h>
16	#include <linux/file.h>
17	#include <linux/anon_inodes.h>
18	#include <linux/fs.h>	16	#include <linux/fs.h>
19	#include <linux/idr.h>	17	#include <linux/idr.h>
20	#include <linux/iommu.h>	18	#include <linux/iommu.h>
@@ -35,6 +33,7 @@ · 14 unmodified lines
35	#include <linux/pm_runtime.h>	33	#include <linux/pm_runtime.h>
36	#include <linux/interval_tree.h>	34	#include <linux/interval_tree.h>
37	#include <linux/iova_bitmap.h>	35	#include <linux/iova_bitmap.h>
		36	#include <linux/iommufd.h>
38	#include "vfio.h"	37	#include "vfio.h"
39		38
40	#define DRIVER_VERSION "0.3"	39	#define DRIVER_VERSION "0.3"
@@ -42,17 +41,11 @@ · 1 unmodified line
42	#define DRIVER_DESC "VFIO - User Level meta-driver"	41	#define DRIVER_DESC "VFIO - User Level meta-driver"
43		42
44	static struct vfio {	43	static struct vfio {
45	struct class *class;
46	struct list_head group_list;
47	struct mutex group_lock; /* locks group_list */
48	struct ida group_ida;
49	dev_t group_devt;
50	struct class *device_class;	44	struct class *device_class;
51	struct ida device_ida;	45	struct ida device_ida;
52	} vfio;	46	} vfio;
53		47
54	static DEFINE_XARRAY(vfio_device_set_xa);	48	static DEFINE_XARRAY(vfio_device_set_xa);
55	static const struct file_operations vfio_group_fops;
56		49
57	int vfio_assign_device_set(struct vfio_device device, void set_id)	50	int vfio_assign_device_set(struct vfio_device device, void set_id)
58	{	51	{
@@ -138,208 +131,21 @@ unsigned int vfio_device_set_open_count(struct vfio_device_set *dev_set) · 79 unmodified lines
138	}	131	}
139	EXPORT_SYMBOL_GPL(vfio_device_set_open_count);	132	EXPORT_SYMBOL_GPL(vfio_device_set_open_count);
140		133
141	/*
142	* Group objects - create, release, get, put, search
143	*/
144	static struct vfio_group *
145	__vfio_group_get_from_iommu(struct iommu_group *iommu_group)
146	{
147	struct vfio_group *group;
148
149	/*
150	* group->iommu_group from the vfio.group_list cannot be NULL
151	* under the vfio.group_lock.
152	*/
153	list_for_each_entry(group, &vfio.group_list, vfio_next) {
154	if (group->iommu_group == iommu_group) {
155	refcount_inc(&group->drivers);
156	return group;
157	}
158	}
159	return NULL;
160	}
161
162	static struct vfio_group *
163	vfio_group_get_from_iommu(struct iommu_group *iommu_group)
164	{
165	struct vfio_group *group;
166
167	mutex_lock(&vfio.group_lock);
168	group = __vfio_group_get_from_iommu(iommu_group);
169	mutex_unlock(&vfio.group_lock);
170	return group;
171	}
172
173	static void vfio_group_release(struct device *dev)
174	{
175	struct vfio_group *group = container_of(dev, struct vfio_group, dev);
176
177	mutex_destroy(&group->device_lock);
178	mutex_destroy(&group->group_lock);
179	WARN_ON(group->iommu_group);
180	ida_free(&vfio.group_ida, MINOR(group->dev.devt));
181	kfree(group);
182	}
183
184	static struct vfio_group vfio_group_alloc(struct iommu_group iommu_group,
185	enum vfio_group_type type)
186	{
187	struct vfio_group *group;
188	int minor;
189
190	group = kzalloc(sizeof(*group), GFP_KERNEL);
191	if (!group)
192	return ERR_PTR(-ENOMEM);
193
194	minor = ida_alloc_max(&vfio.group_ida, MINORMASK, GFP_KERNEL);
195	if (minor < 0) {
196	kfree(group);
197	return ERR_PTR(minor);
198	}
199
200	device_initialize(&group->dev);
201	group->dev.devt = MKDEV(MAJOR(vfio.group_devt), minor);
202	group->dev.class = vfio.class;
203	group->dev.release = vfio_group_release;
204	cdev_init(&group->cdev, &vfio_group_fops);
205	group->cdev.owner = THIS_MODULE;
206
207	refcount_set(&group->drivers, 1);
208	mutex_init(&group->group_lock);
209	INIT_LIST_HEAD(&group->device_list);
210	mutex_init(&group->device_lock);
211	group->iommu_group = iommu_group;
212	/* put in vfio_group_release() */
213	iommu_group_ref_get(iommu_group);
214	group->type = type;
215	BLOCKING_INIT_NOTIFIER_HEAD(&group->notifier);
216
217	return group;
218	}
219
220	static struct vfio_group vfio_create_group(struct iommu_group iommu_group,
221	enum vfio_group_type type)
222	{
223	struct vfio_group *group;
224	struct vfio_group *ret;
225	int err;
226
227	group = vfio_group_alloc(iommu_group, type);
228	if (IS_ERR(group))
229	return group;
230
231	err = dev_set_name(&group->dev, "%s%d",
232	group->type == VFIO_NO_IOMMU ? "noiommu-" : "",
233	iommu_group_id(iommu_group));
234	if (err) {
235	ret = ERR_PTR(err);
236	goto err_put;
237	}
238
239	mutex_lock(&vfio.group_lock);
240
241	/* Did we race creating this group? */
242	ret = __vfio_group_get_from_iommu(iommu_group);
243	if (ret)
244	goto err_unlock;
245
246	err = cdev_device_add(&group->cdev, &group->dev);
247	if (err) {
248	ret = ERR_PTR(err);
249	goto err_unlock;
250	}
251
252	list_add(&group->vfio_next, &vfio.group_list);
253
254	mutex_unlock(&vfio.group_lock);
255	return group;
256
257	err_unlock:
258	mutex_unlock(&vfio.group_lock);
259	err_put:
260	put_device(&group->dev);
261	return ret;
262	}
263
264	static void vfio_device_remove_group(struct vfio_device *device)
265	{
266	struct vfio_group *group = device->group;
267	struct iommu_group *iommu_group;
268
269	if (group->type == VFIO_NO_IOMMU \|\| group->type == VFIO_EMULATED_IOMMU)
270	iommu_group_remove_device(device->dev);
271
272	/* Pairs with vfio_create_group() / vfio_group_get_from_iommu() */
273	if (!refcount_dec_and_mutex_lock(&group->drivers, &vfio.group_lock))
274	return;
275	list_del(&group->vfio_next);
276
277	/*
278	* We could concurrently probe another driver in the group that might
279	* race vfio_device_remove_group() with vfio_get_group(), so we have to
280	* ensure that the sysfs is all cleaned up under lock otherwise the
281	* cdev_device_add() will fail due to the name aready existing.
282	*/
283	cdev_device_del(&group->cdev, &group->dev);
284
285	mutex_lock(&group->group_lock);
286	/*
287	* These data structures all have paired operations that can only be
288	* undone when the caller holds a live reference on the device. Since
289	* all pairs must be undone these WARN_ON's indicate some caller did not
290	* properly hold the group reference.
291	*/
292	WARN_ON(!list_empty(&group->device_list));
293	WARN_ON(group->notifier.head);
294
295	/*
296	* Revoke all users of group->iommu_group. At this point we know there
297	* are no devices active because we are unplugging the last one. Setting
298	* iommu_group to NULL blocks all new users.
299	*/
300	if (group->container)
301	vfio_group_detach_container(group);
302	iommu_group = group->iommu_group;
303	group->iommu_group = NULL;
304	mutex_unlock(&group->group_lock);
305	mutex_unlock(&vfio.group_lock);
306
307	iommu_group_put(iommu_group);
308	put_device(&group->dev);
309	}
310
311	/*	134	/*
312	* Device objects - create, release, get, put, search	135	* Device objects - create, release, get, put, search
313	*/	136	*/
314	/* Device reference always implies a group reference */	137	/* Device reference always implies a group reference */
315	static void vfio_device_put_registration(struct vfio_device *device)	138	void vfio_device_put_registration(struct vfio_device *device)
316	{	139	{
317	if (refcount_dec_and_test(&device->refcount))	140	if (refcount_dec_and_test(&device->refcount))
318	complete(&device->comp);	141	complete(&device->comp);
319	}	142	}
320		143
321	static bool vfio_device_try_get_registration(struct vfio_device *device)	144	bool vfio_device_try_get_registration(struct vfio_device *device)
322	{	145	{
323	return refcount_inc_not_zero(&device->refcount);	146	return refcount_inc_not_zero(&device->refcount);
324	}	147	}
325		148
326	static struct vfio_device vfio_group_get_device(struct vfio_group group,
327	struct device *dev)
328	{
329	struct vfio_device *device;
330
331	mutex_lock(&group->device_lock);
332	list_for_each_entry(device, &group->device_list, group_next) {
333	if (device->dev == dev &&
334	vfio_device_try_get_registration(device)) {
335	mutex_unlock(&group->device_lock);
336	return device;
337	}
338	}
339	mutex_unlock(&group->device_lock);
340	return NULL;
341	}
342
343	/*	149	/*
344	* VFIO driver API	150	* VFIO driver API
345	*/	151	*/
@@ -448,94 +254,15 @@ void vfio_free_device(struct vfio_device *device) · 102 unmodified lines
448	}	254	}
449	EXPORT_SYMBOL_GPL(vfio_free_device);	255	EXPORT_SYMBOL_GPL(vfio_free_device);
450		256
451	static struct vfio_group vfio_noiommu_group_alloc(struct device dev,
452	enum vfio_group_type type)
453	{
454	struct iommu_group *iommu_group;
455	struct vfio_group *group;
456	int ret;
457
458	iommu_group = iommu_group_alloc();
459	if (IS_ERR(iommu_group))
460	return ERR_CAST(iommu_group);
461
462	ret = iommu_group_set_name(iommu_group, "vfio-noiommu");
463	if (ret)
464	goto out_put_group;
465	ret = iommu_group_add_device(iommu_group, dev);
466	if (ret)
467	goto out_put_group;
468
469	group = vfio_create_group(iommu_group, type);
470	if (IS_ERR(group)) {
471	ret = PTR_ERR(group);
472	goto out_remove_device;
473	}
474	iommu_group_put(iommu_group);
475	return group;
476
477	out_remove_device:
478	iommu_group_remove_device(dev);
479	out_put_group:
480	iommu_group_put(iommu_group);
481	return ERR_PTR(ret);
482	}
483
484	static struct vfio_group vfio_group_find_or_alloc(struct device dev)
485	{
486	struct iommu_group *iommu_group;
487	struct vfio_group *group;
488
489	iommu_group = iommu_group_get(dev);
490	if (!iommu_group && vfio_noiommu) {
491	/*
492	* With noiommu enabled, create an IOMMU group for devices that
493	* don't already have one, implying no IOMMU hardware/driver
494	* exists. Taint the kernel because we're about to give a DMA
495	* capable device to a user without IOMMU protection.
496	*/
497	group = vfio_noiommu_group_alloc(dev, VFIO_NO_IOMMU);
498	if (!IS_ERR(group)) {
499	add_taint(TAINT_USER, LOCKDEP_STILL_OK);
500	dev_warn(dev, "Adding kernel taint for vfio-noiommu group on device\n");
501	}
502	return group;
503	}
504
505	if (!iommu_group)
506	return ERR_PTR(-EINVAL);
507
508	/*
509	* VFIO always sets IOMMU_CACHE because we offer no way for userspace to
510	* restore cache coherency. It has to be checked here because it is only
511	* valid for cases where we are using iommu groups.
512	*/
513	if (!device_iommu_capable(dev, IOMMU_CAP_CACHE_COHERENCY)) {
514	iommu_group_put(iommu_group);
515	return ERR_PTR(-EINVAL);
516	}
517
518	group = vfio_group_get_from_iommu(iommu_group);
519	if (!group)
520	group = vfio_create_group(iommu_group, VFIO_IOMMU);
521
522	/* The vfio_group holds a reference to the iommu_group */
523	iommu_group_put(iommu_group);
524	return group;
525	}
526
527	static int __vfio_register_dev(struct vfio_device *device,	257	static int __vfio_register_dev(struct vfio_device *device,
528	struct vfio_group *group)	258	enum vfio_group_type type)
529	{	259	{
530	struct vfio_device *existing_device;
531	int ret;	260	int ret;
532		261
533	/*	262	if (WARN_ON(device->ops->bind_iommufd &&
534	* In all cases group is the output of one of the group allocation	263	(!device->ops->unbind_iommufd \|\|
535	* functions and we have group->drivers incremented for us.	264	!device->ops->attach_ioas)))
536	*/	265	return -EINVAL;
537	if (IS_ERR(group))
538	return PTR_ERR(group);
539		266
540	/*	267	/*
541	* If the driver doesn't specify a set then the device is added to a	268	* If the driver doesn't specify a set then the device is added to a
@@ -544,25 +271,13 @@ static int __vfio_register_dev(struct vfio_device *device, · 2 unmodified lines
544	if (!device->dev_set)	271	if (!device->dev_set)
545	vfio_assign_device_set(device, device);	272	vfio_assign_device_set(device, device);
546		273
547	existing_device = vfio_group_get_device(group, device->dev);
548	if (existing_device) {
549	/*
550	* group->iommu_group is non-NULL because we hold the drivers
551	* refcount.
552	*/
553	dev_WARN(device->dev, "Device already exists on group %d\n",
554	iommu_group_id(group->iommu_group));
555	vfio_device_put_registration(existing_device);
556	ret = -EBUSY;
557	goto err_out;
558	}
559
560	/* Our reference on group is moved to the device */
561	device->group = group;
562
563	ret = dev_set_name(&device->device, "vfio%d", device->index);	274	ret = dev_set_name(&device->device, "vfio%d", device->index);
564	if (ret)	275	if (ret)
565	goto err_out;	276	return ret;
		277
		278	ret = vfio_device_set_group(device, type);
		279	if (ret)
		280	return ret;
566		281
567	ret = device_add(&device->device);	282	ret = device_add(&device->device);
568	if (ret)	283	if (ret)
@@ -571,9 +286,7 @@ static int __vfio_register_dev(struct vfio_device *device, · 2 unmodified lines
571	/* Refcounting can't start until the driver calls register */	286	/* Refcounting can't start until the driver calls register */
572	refcount_set(&device->refcount, 1);	287	refcount_set(&device->refcount, 1);
573		288
574	mutex_lock(&group->device_lock);	289	vfio_device_group_register(device);
575	list_add(&device->group_next, &group->device_list);
576	mutex_unlock(&group->device_lock);
577		290
578	return 0;	291	return 0;
579	err_out:	292	err_out:
@@ -583,8 +296,7 @@ err_out: · 3 unmodified lines
583		296
584	int vfio_register_group_dev(struct vfio_device *device)	297	int vfio_register_group_dev(struct vfio_device *device)
585	{	298	{
586	return __vfio_register_dev(device,	299	return __vfio_register_dev(device, VFIO_IOMMU);
587	vfio_group_find_or_alloc(device->dev));
588	}	300	}
589	EXPORT_SYMBOL_GPL(vfio_register_group_dev);	301	EXPORT_SYMBOL_GPL(vfio_register_group_dev);
590		302
@@ -594,46 +306,15 @@ EXPORT_SYMBOL_GPL(vfio_register_group_dev); · 3 unmodified lines
594	*/	306	*/
595	int vfio_register_emulated_iommu_dev(struct vfio_device *device)	307	int vfio_register_emulated_iommu_dev(struct vfio_device *device)
596	{	308	{
597	return __vfio_register_dev(device,	309	return __vfio_register_dev(device, VFIO_EMULATED_IOMMU);
598	vfio_noiommu_group_alloc(device->dev, VFIO_EMULATED_IOMMU));
599	}	310	}
600	EXPORT_SYMBOL_GPL(vfio_register_emulated_iommu_dev);	311	EXPORT_SYMBOL_GPL(vfio_register_emulated_iommu_dev);
601		312
602	static struct vfio_device vfio_device_get_from_name(struct vfio_group group,
603	char *buf)
604	{
605	struct vfio_device it, device = ERR_PTR(-ENODEV);
606
607	mutex_lock(&group->device_lock);
608	list_for_each_entry(it, &group->device_list, group_next) {
609	int ret;
610
611	if (it->ops->match) {
612	ret = it->ops->match(it, buf);
613	if (ret < 0) {
614	device = ERR_PTR(ret);
615	break;
616	}
617	} else {
618	ret = !strcmp(dev_name(it->dev), buf);
619	}
620
621	if (ret && vfio_device_try_get_registration(it)) {
622	device = it;
623	break;
624	}
625	}
626	mutex_unlock(&group->device_lock);
627
628	return device;
629	}
630
631	/*	313	/*
632	* Decrement the device reference count and wait for the device to be	314	* Decrement the device reference count and wait for the device to be
633	* removed. Open file descriptors for the device... */	315	* removed. Open file descriptors for the device... */
634	void vfio_unregister_group_dev(struct vfio_device *device)	316	void vfio_unregister_group_dev(struct vfio_device *device)
635	{	317	{
636	struct vfio_group *group = device->group;
637	unsigned int i = 0;	318	unsigned int i = 0;
638	bool interrupted = false;	319	bool interrupted = false;
639	long rc;	320	long rc;
@@ -661,333 +342,101 @@ void vfio_unregister_group_dev(struct vfio_device *device) · 21 unmodified lines
661	}	342	}
662	}	343	}
663		344
664	mutex_lock(&group->device_lock);	345	vfio_device_group_unregister(device);
665	list_del(&device->group_next);
666	mutex_unlock(&group->device_lock);
667		346
668	/* Balances device_add in register path */	347	/* Balances device_add in register path */
669	device_del(&device->device);	348	device_del(&device->device);
670		349
		350	/* Balances vfio_device_set_group in register path */
671	vfio_device_remove_group(device);	351	vfio_device_remove_group(device);
672	}	352	}
673	EXPORT_SYMBOL_GPL(vfio_unregister_group_dev);	353	EXPORT_SYMBOL_GPL(vfio_unregister_group_dev);
674		354
675	/*	355	/* true if the vfio_device has open_device() called but not close_device() */
676	* VFIO Group fd, /dev/vfio/$GROUP	356	static bool vfio_assert_device_open(struct vfio_device *device)
677	*/
678	/*
679	* VFIO_GROUP_UNSET_CONTAINER should fail if there are other users or
680	* if there was no container to unset. Since the ioctl is called on
681	* the group, we know that still exists, therefore the only valid
682	* transition here is 1->0.
683	*/
684	static int vfio_group_ioctl_unset_container(struct vfio_group *group)
685	{	357	{
686	int ret = 0;	358	return !WARN_ON_ONCE(!READ_ONCE(device->open_count));
687
688	mutex_lock(&group->group_lock);
689	if (!group->container) {
690	ret = -EINVAL;
691	goto out_unlock;
692	}
693	if (group->container_users != 1) {
694	ret = -EBUSY;
695	goto out_unlock;
696	}
697	vfio_group_detach_container(group);
698
699	out_unlock:
700	mutex_unlock(&group->group_lock);
701	return ret;
702	}	359	}
703		360
704	static int vfio_group_ioctl_set_container(struct vfio_group *group,	361	static int vfio_device_first_open(struct vfio_device *device,
705	int __user *arg)	362	struct iommufd_ctx iommufd, struct kvm kvm)
706	{	363	{
707	struct vfio_container *container;
708	struct fd f;
709	int ret;	364	int ret;
710	int fd;
711		365
712	if (get_user(fd, arg))	366	lockdep_assert_held(&device->dev_set->lock);
713	return -EFAULT;
714		367
715	f = fdget(fd);	368	if (!try_module_get(device->dev->driver->owner))
716	if (!f.file)	369	return -ENODEV;
717	return -EBADF;
718		370
719	mutex_lock(&group->group_lock);	371	if (iommufd)
720	if (group->container \|\| WARN_ON(group->container_users)) {	372	ret = vfio_iommufd_bind(device, iommufd);
721	ret = -EINVAL;	373	else
722	goto out_unlock;	374	ret = vfio_device_group_use_iommu(device);
723	}	375	if (ret)
724	if (!group->iommu_group) {	376	goto err_module_put;
725	ret = -ENODEV;
726	goto out_unlock;
727	}
728		377
729	container = vfio_container_from_file(f.file);	378	device->kvm = kvm;
730	ret = -EINVAL;	379	if (device->ops->open_device) {
731	if (container) {	380	ret = device->ops->open_device(device);
732	ret = vfio_container_attach_group(container, group);	381	if (ret)
733	goto out_unlock;	382	goto err_unuse_iommu;
734	}	383	}
		384	return 0;
735		385
736	out_unlock:	386	err_unuse_iommu:
737	mutex_unlock(&group->group_lock);	387	device->kvm = NULL;
738	fdput(f);	388	if (iommufd)
		389	vfio_iommufd_unbind(device);
		390	else
		391	vfio_device_group_unuse_iommu(device);
		392	err_module_put:
		393	module_put(device->dev->driver->owner);
739	return ret;	394	return ret;
740	}	395	}
741		396
742	static const struct file_operations vfio_device_fops;	397	static void vfio_device_last_close(struct vfio_device *device,
743		398	struct iommufd_ctx *iommufd)
744	/* true if the vfio_device has open_device() called but not close_device() */
745	bool vfio_assert_device_open(struct vfio_device *device)
746	{	399	{
747	return !WARN_ON_ONCE(!READ_ONCE(device->open_count));	400	lockdep_assert_held(&device->dev_set->lock);
		401
		402	if (device->ops->close_device)
		403	device->ops->close_device(device);
		404	device->kvm = NULL;
		405	if (iommufd)
		406	vfio_iommufd_unbind(device);
		407	else
		408	vfio_device_group_unuse_iommu(device);
		409	module_put(device->dev->driver->owner);
748	}	410	}
749		411
750	static struct file vfio_device_open(struct vfio_device device)	412	int vfio_device_open(struct vfio_device *device,
		413	struct iommufd_ctx iommufd, struct kvm kvm)
751	{	414	{
752	struct file *filep;	415	int ret = 0;
753	int ret;
754
755	mutex_lock(&device->group->group_lock);
756	ret = vfio_device_assign_container(device);
757	mutex_unlock(&device->group->group_lock);
758	if (ret)
759	return ERR_PTR(ret);
760
761	if (!try_module_get(device->dev->driver->owner)) {
762	ret = -ENODEV;
763	goto err_unassign_container;
764	}
765		416
766	mutex_lock(&device->dev_set->lock);	417	mutex_lock(&device->dev_set->lock);
767	device->open_count++;	418	device->open_count++;
768	if (device->open_count == 1) {	419	if (device->open_count == 1) {
769	/*	420	ret = vfio_device_first_open(device, iommufd, kvm);
770	* Here we pass the KVM pointer with the group under the read	421	if (ret)
771	* lock. If the device driver will use it, it must obtain a	422	device->open_count--;
772	* reference and release it during close_device.
773	*/
774	mutex_lock(&device->group->group_lock);
775	device->kvm = device->group->kvm;
776
777	if (device->ops->open_device) {
778	ret = device->ops->open_device(device);
779	if (ret)
780	goto err_undo_count;
781	}
782	vfio_device_container_register(device);
783	mutex_unlock(&device->group->group_lock);
784	}
785	mutex_unlock(&device->dev_set->lock);
786
787	/*
788	* We can't use anon_inode_getfd() because we need to modify
789	* the f_mode flags directly to allow more than just ioctls
790	*/
791	filep = anon_inode_getfile("[vfio-device]", &vfio_device_fops,
792	device, O_RDWR);
793	if (IS_ERR(filep)) {
794	ret = PTR_ERR(filep);
795	goto err_close_device;
796	}
797
798	/*
799	* TODO: add an anon_inode interface to do this.
800	* Appears to be missing by lack of need rather than
801	* explicitly prevented. Now there's need.
802	*/
803	filep->f_mode \|= (FMODE_PREAD \| FMODE_PWRITE);
804
805	if (device->group->type == VFIO_NO_IOMMU)
806	dev_warn(device->dev, "vfio-noiommu device opened by user "
807	"(%s:%d)\n", current->comm, task_pid_nr(current));
808	/*
809	* On success the ref of device is moved to the file and
810	* put in vfio_device_fops_release()
811	*/
812	return filep;
813
814	err_close_device:
815	mutex_lock(&device->dev_set->lock);
816	mutex_lock(&device->group->group_lock);
817	if (device->open_count == 1) {
818	if (device->ops->close_device)
819	device->ops->close_device(device);
820
821	vfio_device_container_unregister(device);
822	}	423	}
823	err_undo_count:
824	mutex_unlock(&device->group->group_lock);
825	device->open_count--;
826	if (device->open_count == 0 && device->kvm)
827	device->kvm = NULL;
828	mutex_unlock(&device->dev_set->lock);	424	mutex_unlock(&device->dev_set->lock);
829	module_put(device->dev->driver->owner);
830	err_unassign_container:
831	vfio_device_unassign_container(device);
832	return ERR_PTR(ret);
833	}
834
835	static int vfio_group_ioctl_get_device_fd(struct vfio_group *group,
836	char __user *arg)
837	{
838	struct vfio_device *device;
839	struct file *filep;
840	char *buf;
841	int fdno;
842	int ret;
843
844	buf = strndup_user(arg, PAGE_SIZE);
845	if (IS_ERR(buf))
846	return PTR_ERR(buf);
847
848	device = vfio_device_get_from_name(group, buf);
849	kfree(buf);
850	if (IS_ERR(device))
851	return PTR_ERR(device);
852
853	fdno = get_unused_fd_flags(O_CLOEXEC);
854	if (fdno < 0) {
855	ret = fdno;
856	goto err_put_device;
857	}
858
859	filep = vfio_device_open(device);
860	if (IS_ERR(filep)) {
861	ret = PTR_ERR(filep);
862	goto err_put_fdno;
863	}
864
865	fd_install(fdno, filep);
866	return fdno;
867
868	err_put_fdno:
869	put_unused_fd(fdno);
870	err_put_device:
871	vfio_device_put_registration(device);
872	return ret;
873	}
874
875	static int vfio_group_ioctl_get_status(struct vfio_group *group,
876	struct vfio_group_status __user *arg)
877	{
878	unsigned long minsz = offsetofend(struct vfio_group_status, flags);
879	struct vfio_group_status status;
880
881	if (copy_from_user(&status, arg, minsz))
882	return -EFAULT;
883
884	if (status.argsz < minsz)
885	return -EINVAL;
886
887	status.flags = 0;
888
889	mutex_lock(&group->group_lock);
890	if (!group->iommu_group) {
891	mutex_unlock(&group->group_lock);
892	return -ENODEV;
893	}
894
895	if (group->container)
896	status.flags \|= VFIO_GROUP_FLAGS_CONTAINER_SET \|
897	VFIO_GROUP_FLAGS_VIABLE;
898	else if (!iommu_group_dma_owner_claimed(group->iommu_group))
899	status.flags \|= VFIO_GROUP_FLAGS_VIABLE;
900	mutex_unlock(&group->group_lock);
901
902	if (copy_to_user(arg, &status, minsz))
903	return -EFAULT;
904	return 0;
905	}
906
907	static long vfio_group_fops_unl_ioctl(struct file *filep,
908	unsigned int cmd, unsigned long arg)
909	{
910	struct vfio_group *group = filep->private_data;
911	void __user uarg = (void __user )arg;
912
913	switch (cmd) {
914	case VFIO_GROUP_GET_DEVICE_FD:
915	return vfio_group_ioctl_get_device_fd(group, uarg);
916	case VFIO_GROUP_GET_STATUS:
917	return vfio_group_ioctl_get_status(group, uarg);
918	case VFIO_GROUP_SET_CONTAINER:
919	return vfio_group_ioctl_set_container(group, uarg);
920	case VFIO_GROUP_UNSET_CONTAINER:
921	return vfio_group_ioctl_unset_container(group);
922	default:
923	return -ENOTTY;
924	}
925	}
926
927	static int vfio_group_fops_open(struct inode inode, struct file filep)
928	{
929	struct vfio_group *group =
930	container_of(inode->i_cdev, struct vfio_group, cdev);
931	int ret;
932
933	mutex_lock(&group->group_lock);
934		425
935	/*
936	* drivers can be zero if this races with vfio_device_remove_group(), it
937	* will be stable at 0 under the group rwsem
938	*/
939	if (refcount_read(&group->drivers) == 0) {
940	ret = -ENODEV;
941	goto out_unlock;
942	}
943
944	if (group->type == VFIO_NO_IOMMU && !capable(CAP_SYS_RAWIO)) {
945	ret = -EPERM;
946	goto out_unlock;
947	}
948
949	/*
950	* Do we need multiple instances of the group open? Seems not.
951	*/
952	if (group->opened_file) {
953	ret = -EBUSY;
954	goto out_unlock;
955	}
956	group->opened_file = filep;
957	filep->private_data = group;
958	ret = 0;
959	out_unlock:
960	mutex_unlock(&group->group_lock);
961	return ret;	426	return ret;
962	}	427	}
963		428
964	static int vfio_group_fops_release(struct inode inode, struct file filep)	429	void vfio_device_close(struct vfio_device *device,
		430	struct iommufd_ctx *iommufd)
965	{	431	{
966	struct vfio_group *group = filep->private_data;	432	mutex_lock(&device->dev_set->lock);
967		433	vfio_assert_device_open(device);
968	filep->private_data = NULL;	434	if (device->open_count == 1)
969		435	vfio_device_last_close(device, iommufd);
970	mutex_lock(&group->group_lock);	436	device->open_count--;
971	/*	437	mutex_unlock(&device->dev_set->lock);
972	* Device FDs hold a group file reference, therefore the group release
973	* is only called when there are no open devices.
974	*/
975	WARN_ON(group->notifier.head);
976	if (group->container)
977	vfio_group_detach_container(group);
978	group->opened_file = NULL;
979	mutex_unlock(&group->group_lock);
980	return 0;
981	}	438	}
982		439
983	static const struct file_operations vfio_group_fops = {
984	.owner = THIS_MODULE,
985	.unlocked_ioctl = vfio_group_fops_unl_ioctl,
986	.compat_ioctl = compat_ptr_ioctl,
987	.open = vfio_group_fops_open,
988	.release = vfio_group_fops_release,
989	};
990
991	/*	440	/*
992	* Wrapper around pm_runtime_resume_and_get().	441	* Wrapper around pm_runtime_resume_and_get().
993	* Return error code on failure or 0 on success.	442	* Return error code on failure or 0 on success.
@@ -1028,24 +477,7 @@ static int vfio_device_fops_release(struct inode inode, struct file filep) · 34 unmodified lines
1028	{	477	{
1029	struct vfio_device *device = filep->private_data;	478	struct vfio_device *device = filep->private_data;
1030		479
1031	mutex_lock(&device->dev_set->lock);	480	vfio_device_group_close(device);
1032	vfio_assert_device_open(device);
1033	mutex_lock(&device->group->group_lock);
1034	if (device->open_count == 1) {
1035	if (device->ops->close_device)
1036	device->ops->close_device(device);
1037
1038	vfio_device_container_unregister(device);
1039	}
1040	mutex_unlock(&device->group->group_lock);
1041	device->open_count--;
1042	if (device->open_count == 0)
1043	device->kvm = NULL;
1044	mutex_unlock(&device->dev_set->lock);
1045
1046	module_put(device->dev->driver->owner);
1047
1048	vfio_device_unassign_container(device);
1049		481
1050	vfio_device_put_registration(device);	482	vfio_device_put_registration(device);
1051		483
@@ -1568,7 +1000,7 @@ static int vfio_device_fops_mmap(struct file filep, struct vm_area_struct vma) · 516 unmodified lines
1568	return device->ops->mmap(device, vma);	1000	return device->ops->mmap(device, vma);
1569	}	1001	}
1570		1002
1571	static const struct file_operations vfio_device_fops = {	1003	const struct file_operations vfio_device_fops = {
1572	.owner = THIS_MODULE,	1004	.owner = THIS_MODULE,
1573	.release = vfio_device_fops_release,	1005	.release = vfio_device_fops_release,
1574	.read = vfio_device_fops_read,	1006	.read = vfio_device_fops_read,
@@ -1578,118 +1010,6 @@ static const struct file_operations vfio_device_fops = { · 3 unmodified lines
1578	.mmap = vfio_device_fops_mmap,	1010	.mmap = vfio_device_fops_mmap,
1579	};	1011	};
1580		1012
1581	/**
1582	* vfio_file_iommu_group - Return the struct iommu_group for the vfio group file
1583	* @file: VFIO group file
1584	*
1585	* The returned iommu_group is valid as long as a ref is held on the file. This
1586	* returns a reference on the group. This function is deprecated, only the SPAPR
1587	* path in kvm should call it.
1588	*/
1589	struct iommu_group vfio_file_iommu_group(struct file file)
1590	{
1591	struct vfio_group *group = file->private_data;
1592	struct iommu_group *iommu_group = NULL;
1593
1594	if (!IS_ENABLED(CONFIG_SPAPR_TCE_IOMMU))
1595	return NULL;
1596
1597	if (!vfio_file_is_group(file))
1598	return NULL;
1599
1600	mutex_lock(&group->group_lock);
1601	if (group->iommu_group) {
1602	iommu_group = group->iommu_group;
1603	iommu_group_ref_get(iommu_group);
1604	}
1605	mutex_unlock(&group->group_lock);
1606	return iommu_group;
1607	}
1608	EXPORT_SYMBOL_GPL(vfio_file_iommu_group);
1609
1610	/**
1611	* vfio_file_is_group - True if the file is usable with VFIO aPIS
1612	* @file: VFIO group file
1613	*/
1614	bool vfio_file_is_group(struct file *file)
1615	{
1616	return file->f_op == &vfio_group_fops;
1617	}
1618	EXPORT_SYMBOL_GPL(vfio_file_is_group);
1619
1620	/**
1621	* vfio_file_enforced_coherent - True if the DMA associated with the VFIO file
1622	* is always CPU cache coherent
1623	* @file: VFIO group file
1624	*
1625	* Enforced coherency means that the IOMMU ignores things like the PCIe no-snoop
1626	* bit in DMA transactions. A return of false indicates that the user has
1627	* rights to access additional instructions such as wbinvd on x86.
1628	*/
1629	bool vfio_file_enforced_coherent(struct file *file)
1630	{
1631	struct vfio_group *group = file->private_data;
1632	bool ret;
1633
1634	if (!vfio_file_is_group(file))
1635	return true;
1636
1637	mutex_lock(&group->group_lock);
1638	if (group->container) {
1639	ret = vfio_container_ioctl_check_extension(group->container,
1640	VFIO_DMA_CC_IOMMU);
1641	} else {
1642	/*
1643	* Since the coherency state is determined only once a container
1644	* is attached the user must do so before they can prove they
1645	* have permission.
1646	*/
1647	ret = true;
1648	}
1649	mutex_unlock(&group->group_lock);
1650	return ret;
1651	}
1652	EXPORT_SYMBOL_GPL(vfio_file_enforced_coherent);
1653
1654	/**
1655	* vfio_file_set_kvm - Link a kvm with VFIO drivers
1656	* @file: VFIO group file
1657	* @kvm: KVM to link
1658	*
1659	* When a VFIO device is first opened the KVM will be available in
1660	* device->kvm if one was associated with the group.
1661	*/
1662	void vfio_file_set_kvm(struct file file, struct kvm kvm)
1663	{
1664	struct vfio_group *group = file->private_data;
1665
1666	if (!vfio_file_is_group(file))
1667	return;
1668
1669	mutex_lock(&group->group_lock);
1670	group->kvm = kvm;
1671	mutex_unlock(&group->group_lock);
1672	}
1673	EXPORT_SYMBOL_GPL(vfio_file_set_kvm);
1674
1675	/**
1676	* vfio_file_has_dev - True if the VFIO file is a handle for device
1677	* @file: VFIO file to check
1678	* @device: Device that must be part of the file
1679	*
1680	* Returns true if given file has permission to manipulate the given device.
1681	*/
1682	bool vfio_file_has_dev(struct file file, struct vfio_device device)
1683	{
1684	struct vfio_group *group = file->private_data;
1685
1686	if (!vfio_file_is_group(file))
1687	return false;
1688
1689	return group == device->group;
1690	}
1691	EXPORT_SYMBOL_GPL(vfio_file_has_dev);
1692
1693	/*	1013	/*
1694	* Sub-module support	1014	* Sub-module support
1695	*/	1015	*/
@@ -1810,35 +1130,136 @@ int vfio_set_irqs_validate_and_prepare(struct vfio_irq_set *hdr, int num_irqs, · 114 unmodified lines
1810	EXPORT_SYMBOL(vfio_set_irqs_validate_and_prepare);	1130	EXPORT_SYMBOL(vfio_set_irqs_validate_and_prepare);
1811		1131
1812	/*	1132	/*
1813	* Module/class support	1133	* Pin contiguous user pages and return their associated host pages for local
		1134	* domain only.
		1135	* @device [in] : device
		1136	* @iova [in] : starting IOVA of user pages to be pinned.
		1137	* @npage [in] : count of pages to be pinned. This count should not
		1138	* be greater than VFIO_PIN_PAGES_MAX_ENTRIES.
		1139	* @prot [in] : protection flags
		1140	* @pages[out] : array of host pages
		1141	* Return error or number of pages pinned.
		1142	*
		1143	* A driver may only call this function if the vfio_device was created
		1144	* by vfio_register_emulated_iommu_dev() due to vfio_device_container_pin_pages().
		1145	*/
		1146	int vfio_pin_pages(struct vfio_device *device, dma_addr_t iova,
		1147	int npage, int prot, struct page **pages)
		1148	{
		1149	/* group->container cannot change while a vfio device is open */
		1150	if (!pages \|\| !npage \|\| WARN_ON(!vfio_assert_device_open(device)))
		1151	return -EINVAL;
		1152	if (vfio_device_has_container(device))
		1153	return vfio_device_container_pin_pages(device, iova,
		1154	npage, prot, pages);
		1155	if (device->iommufd_access) {
		1156	int ret;
		1157
		1158	if (iova > ULONG_MAX)
		1159	return -EINVAL;
		1160	/*
		1161	* VFIO ignores the sub page offset, npages is from the start of
		1162	* a PAGE_SIZE chunk of IOVA. The caller is expected to recover
		1163	* the sub page offset by doing:
		1164	* pages[0] + (iova % PAGE_SIZE)
		1165	*/
		1166	ret = iommufd_access_pin_pages(
		1167	device->iommufd_access, ALIGN_DOWN(iova, PAGE_SIZE),
		1168	npage * PAGE_SIZE, pages,
		1169	(prot & IOMMU_WRITE) ? IOMMUFD_ACCESS_RW_WRITE : 0);
		1170	if (ret)
		1171	return ret;
		1172	return npage;
		1173	}
		1174	return -EINVAL;
		1175	}
		1176	EXPORT_SYMBOL(vfio_pin_pages);
		1177
		1178	/*
		1179	* Unpin contiguous host pages for local domain only.
		1180	* @device [in] : device
		1181	* @iova [in] : starting address of user pages to be unpinned.
		1182	* @npage [in] : count of pages to be unpinned. This count should not
		1183	* be greater than VFIO_PIN_PAGES_MAX_ENTRIES.
		1184	*/
		1185	void vfio_unpin_pages(struct vfio_device *device, dma_addr_t iova, int npage)
		1186	{
		1187	if (WARN_ON(!vfio_assert_device_open(device)))
		1188	return;
		1189
		1190	if (vfio_device_has_container(device)) {
		1191	vfio_device_container_unpin_pages(device, iova, npage);
		1192	return;
		1193	}
		1194	if (device->iommufd_access) {
		1195	if (WARN_ON(iova > ULONG_MAX))
		1196	return;
		1197	iommufd_access_unpin_pages(device->iommufd_access,
		1198	ALIGN_DOWN(iova, PAGE_SIZE),
		1199	npage * PAGE_SIZE);
		1200	return;
		1201	}
		1202	}
		1203	EXPORT_SYMBOL(vfio_unpin_pages);
		1204
		1205	/*
		1206	* This interface allows the CPUs to perform some sort of virtual DMA on
		1207	* behalf of the device.
		1208	*
		1209	* CPUs read/write from/into a range of IOVAs pointing to user space memory
		1210	* into/from a kernel buffer.
		1211	*
		1212	* As the read/write of user space memory is conducted via the CPUs and is
		1213	* not a real device DMA, it is not necessary to pin the user space memory.
		1214	*
		1215	* @device [in] : VFIO device
		1216	* @iova [in] : base IOVA of a user space buffer
		1217	* @data [in] : pointer to kernel buffer
		1218	* @len [in] : kernel buffer length
		1219	* @write : indicate read or write
		1220	* Return error code on failure or 0 on success.
1814	*/	1221	*/
1815	static char vfio_devnode(struct device dev, umode_t *mode)	1222	int vfio_dma_rw(struct vfio_device device, dma_addr_t iova, void data,
		1223	size_t len, bool write)
1816	{	1224	{
1817	return kasprintf(GFP_KERNEL, "vfio/%s", dev_name(dev));	1225	if (!data \|\| len <= 0 \|\| !vfio_assert_device_open(device))
		1226	return -EINVAL;
		1227
		1228	if (vfio_device_has_container(device))
		1229	return vfio_device_container_dma_rw(device, iova,
		1230	data, len, write);
		1231
		1232	if (device->iommufd_access) {
		1233	unsigned int flags = 0;
		1234
		1235	if (iova > ULONG_MAX)
		1236	return -EINVAL;
		1237
		1238	/* VFIO historically tries to auto-detect a kthread */
		1239	if (!current->mm)
		1240	flags \|= IOMMUFD_ACCESS_RW_KTHREAD;
		1241	if (write)
		1242	flags \|= IOMMUFD_ACCESS_RW_WRITE;
		1243	return iommufd_access_rw(device->iommufd_access, iova, data,
		1244	len, flags);
		1245	}
		1246	return -EINVAL;
1818	}	1247	}
		1248	EXPORT_SYMBOL(vfio_dma_rw);
1819		1249
		1250	/*
		1251	* Module/class support
		1252	*/
1820	static int __init vfio_init(void)	1253	static int __init vfio_init(void)
1821	{	1254	{
1822	int ret;	1255	int ret;
1823		1256
1824	ida_init(&vfio.group_ida);
1825	ida_init(&vfio.device_ida);	1257	ida_init(&vfio.device_ida);
1826	mutex_init(&vfio.group_lock);
1827	INIT_LIST_HEAD(&vfio.group_list);
1828		1258
1829	ret = vfio_container_init();	1259	ret = vfio_group_init();
1830	if (ret)	1260	if (ret)
1831	return ret;	1261	return ret;
1832		1262
1833	/* /dev/vfio/$GROUP */
1834	vfio.class = class_create(THIS_MODULE, "vfio");
1835	if (IS_ERR(vfio.class)) {
1836	ret = PTR_ERR(vfio.class);
1837	goto err_group_class;
1838	}
1839
1840	vfio.class->devnode = vfio_devnode;
1841
1842	/* /sys/class/vfio-dev/vfioX */	1263	/* /sys/class/vfio-dev/vfioX */
1843	vfio.device_class = class_create(THIS_MODULE, "vfio-dev");	1264	vfio.device_class = class_create(THIS_MODULE, "vfio-dev");
1844	if (IS_ERR(vfio.device_class)) {	1265	if (IS_ERR(vfio.device_class)) {
@@ -1846,36 +1267,20 @@ static int __init vfio_init(void) · 1 unmodified line
1846	goto err_dev_class;	1267	goto err_dev_class;
1847	}	1268	}
1848		1269
1849	ret = alloc_chrdev_region(&vfio.group_devt, 0, MINORMASK + 1, "vfio");
1850	if (ret)
1851	goto err_alloc_chrdev;
1852
1853	pr_info(DRIVER_DESC " version: " DRIVER_VERSION "\n");	1270	pr_info(DRIVER_DESC " version: " DRIVER_VERSION "\n");
1854	return 0;	1271	return 0;
1855		1272
1856	err_alloc_chrdev:
1857	class_destroy(vfio.device_class);
1858	vfio.device_class = NULL;
1859	err_dev_class:	1273	err_dev_class:
1860	class_destroy(vfio.class);	1274	vfio_group_cleanup();
1861	vfio.class = NULL;
1862	err_group_class:
1863	vfio_container_cleanup();
1864	return ret;	1275	return ret;
1865	}	1276	}
1866		1277
1867	static void __exit vfio_cleanup(void)	1278	static void __exit vfio_cleanup(void)
1868	{	1279	{
1869	WARN_ON(!list_empty(&vfio.group_list));
1870
1871	ida_destroy(&vfio.device_ida);	1280	ida_destroy(&vfio.device_ida);
1872	ida_destroy(&vfio.group_ida);
1873	unregister_chrdev_region(vfio.group_devt, MINORMASK + 1);
1874	class_destroy(vfio.device_class);	1281	class_destroy(vfio.device_class);
1875	vfio.device_class = NULL;	1282	vfio.device_class = NULL;
1876	class_destroy(vfio.class);	1283	vfio_group_cleanup();
1877	vfio_container_cleanup();
1878	vfio.class = NULL;
1879	xa_destroy(&vfio_device_set_xa);	1284	xa_destroy(&vfio_device_set_xa);
1880	}	1285	}
1881		1286
@@ -1886,6 +1291,4 @@ MODULE_VERSION(DRIVER_VERSION); · 4 unmodified lines
1886	MODULE_LICENSE("GPL v2");	1291	MODULE_LICENSE("GPL v2");
1887	MODULE_AUTHOR(DRIVER_AUTHOR);	1292	MODULE_AUTHOR(DRIVER_AUTHOR);
1888	MODULE_DESCRIPTION(DRIVER_DESC);	1293	MODULE_DESCRIPTION(DRIVER_DESC);
1889	MODULE_ALIAS_MISCDEV(VFIO_MINOR);
1890	MODULE_ALIAS("devname:vfio/vfio");
1891	MODULE_SOFTDEP("post: vfio_iommu_type1 vfio_iommu_spapr_tce");	1294	MODULE_SOFTDEP("post: vfio_iommu_type1 vfio_iommu_spapr_tce");

include/linux/intel-svm.h+0−13

@@ -13,17 +13,4 @@ · 12 unmodified lines
13	#define PRQ_RING_MASK ((0x1000 << PRQ_ORDER) - 0x20)	13	#define PRQ_RING_MASK ((0x1000 << PRQ_ORDER) - 0x20)
14	#define PRQ_DEPTH ((0x1000 << PRQ_ORDER) >> 5)	14	#define PRQ_DEPTH ((0x1000 << PRQ_ORDER) >> 5)
15		15
16	/*
17	* The SVM_FLAG_SUPERVISOR_MODE flag requests a PASID which can be used only
18	* for access to kernel addresses. No IOTLB flushes are automatically done
19	* for kernel mappings; it is valid only for access to the kernel's static
20	* 1:1 mapping of physical memory — not to vmalloc or even module mappings.
21	* A future API addition may permit the use of such ranges, by means of an
22	* explicit IOTLB flush call (akin to the DMA API's unmap method).
23	*
24	* It is unlikely that we will ever hook into flush_tlb_kernel_range() to
25	* do such IOTLB flushes automatically.
26	*/
27	#define SVM_FLAG_SUPERVISOR_MODE BIT(0)
28
29	#endif /* __INTEL_SVM_H__ */	16	#endif /* __INTEL_SVM_H__ */

include/linux/interval_tree.h+58−0

@@ -27,4 +27,62 @@ extern struct interval_tree_node * · 26 unmodified lines
27	interval_tree_iter_next(struct interval_tree_node *node,	27	interval_tree_iter_next(struct interval_tree_node *node,
28	unsigned long start, unsigned long last);	28	unsigned long start, unsigned long last);
29		29
		30	/**
		31	* struct interval_tree_span_iter - Find used and unused spans.
		32	* @start_hole: Start of an interval for a hole when is_hole == 1
		33	* @last_hole: Inclusive end of an interval for a hole when is_hole == 1
		34	* @start_used: Start of a used interval when is_hole == 0
		35	* @last_used: Inclusive end of a used interval when is_hole == 0
		36	* @is_hole: 0 == used, 1 == is_hole, -1 == done iteration
		37	*
		38	* This iterator travels over spans in an interval tree. It does not return
		39	* nodes but classifies each span as either a hole, where no nodes intersect, or
		40	* a used, which is fully covered by nodes. Each iteration step toggles between
		41	* hole and used until the entire range is covered. The returned spans always
		42	* fully cover the requested range.
		43	*
		44	* The iterator is greedy, it always returns the largest hole or used possible,
		45	* consolidating all consecutive nodes.
		46	*
		47	* Use interval_tree_span_iter_done() to detect end of iteration.
		48	*/
		49	struct interval_tree_span_iter {
		50	/* private: not for use by the caller */
		51	struct interval_tree_node *nodes[2];
		52	unsigned long first_index;
		53	unsigned long last_index;
		54
		55	/* public: */
		56	union {
		57	unsigned long start_hole;
		58	unsigned long start_used;
		59	};
		60	union {
		61	unsigned long last_hole;
		62	unsigned long last_used;
		63	};
		64	int is_hole;
		65	};
		66
		67	void interval_tree_span_iter_first(struct interval_tree_span_iter *state,
		68	struct rb_root_cached *itree,
		69	unsigned long first_index,
		70	unsigned long last_index);
		71	void interval_tree_span_iter_advance(struct interval_tree_span_iter *iter,
		72	struct rb_root_cached *itree,
		73	unsigned long new_index);
		74	void interval_tree_span_iter_next(struct interval_tree_span_iter *state);
		75
		76	static inline bool
		77	interval_tree_span_iter_done(struct interval_tree_span_iter *state)
		78	{
		79	return state->is_hole == -1;
		80	}
		81
		82	#define interval_tree_for_each_span(span, itree, first_index, last_index) \
		83	for (interval_tree_span_iter_first(span, itree, \
		84	first_index, last_index); \
		85	!interval_tree_span_iter_done(span); \
		86	interval_tree_span_iter_next(span))
		87
30	#endif /* _LINUX_INTERVAL_TREE_H */	88	#endif /* _LINUX_INTERVAL_TREE_H */

include/linux/iommu.h+114−31

 #define __IOMMU_DOMAIN_PT	(1U << 2)  /* Domain is identity mapped   */
 #define __IOMMU_DOMAIN_DMA_FQ	(1U << 3)  /* DMA-API uses flush queue    */
+#define __IOMMU_DOMAIN_SVA	(1U << 4)  /* Shared process address space */
 /*
  * This are the possible domain-types
  *
  *				  certain optimizations for these domains
  *	IOMMU_DOMAIN_DMA_FQ	- As above, but definitely using batched TLB
  *				  invalidation.
+ *	IOMMU_DOMAIN_SVA	- DMA addresses are shared process addresses
+ *				  represented by mm_struct's.
  */
 #define IOMMU_DOMAIN_BLOCKED	(0U)
 #define IOMMU_DOMAIN_IDENTITY	(__IOMMU_DOMAIN_PT)
 #define IOMMU_DOMAIN_DMA_FQ	(__IOMMU_DOMAIN_PAGING |	\
 				 __IOMMU_DOMAIN_DMA_API |	\
 				 __IOMMU_DOMAIN_DMA_FQ)
+#define IOMMU_DOMAIN_SVA	(__IOMMU_DOMAIN_SVA)
 struct iommu_domain {
 	unsigned type;
 	const struct iommu_domain_ops *ops;
 	unsigned long pgsize_bitmap;	/* Bitmap of page sizes in use */
-	iommu_fault_handler_t handler;
-	void *handler_token;
 	struct iommu_domain_geometry geometry;
 	struct iommu_dma_cookie *iova_cookie;
+	enum iommu_page_response_code (*iopf_handler)(struct iommu_fault *fault,
+						      void *data);
+	void *fault_data;
+	union {
+		struct {
+			iommu_fault_handler_t handler;
+			void *handler_token;
+		};
+		struct {	/* IOMMU_DOMAIN_SVA */
+			struct mm_struct *mm;
+			int users;
+		};
+	};
 };
 static inline bool iommu_is_dma_domain(struct iommu_domain *domain)
 	IOMMU_CAP_NOEXEC,		/* IOMMU_NOEXEC flag */
 	IOMMU_CAP_PRE_BOOT_PROTECTION,	/* Firmware says it used the IOMMU for
 					   DMA protection and we should too */
+	/*
+	 * Per-device flag indicating if enforce_cache_coherency() will work on
+	 * this device.
+	 */
+	IOMMU_CAP_ENFORCE_CACHE_COHERENCY,
 };
 /* These are the possible reserved region types */
  *                      driver init to device driver init (default no)
  * @dev_enable/disable_feat: per device entries to enable/disable
  *                               iommu specific features.
- * @sva_bind: Bind process address space to device
- * @sva_unbind: Unbind process address space from device
- * @sva_get_pasid: Get PASID associated to a SVA handle
  * @page_response: handle page request response
  * @def_domain_type: device default domain type, return value:
  *		- IOMMU_DOMAIN_IDENTITY: must use an identity domain
  *		- IOMMU_DOMAIN_DMA: must use a dma domain
  *		- 0: use the default setting
  * @default_domain_ops: the default ops for domains
+ * @remove_dev_pasid: Remove any translation configurations of a specific
+ *                    pasid, so that any DMA transactions with this pasid
+ *                    will be blocked by the hardware.
  * @pgsize_bitmap: bitmap of all possible supported page sizes
  * @owner: Driver module providing these ops
  */
 	int (*dev_enable_feat)(struct device *dev, enum iommu_dev_features f);
 	int (*dev_disable_feat)(struct device *dev, enum iommu_dev_features f);
-	struct iommu_sva *(*sva_bind)(struct device *dev, struct mm_struct *mm,
-				      void *drvdata);
-	void (*sva_unbind)(struct iommu_sva *handle);
-	u32 (*sva_get_pasid)(struct iommu_sva *handle);
 	int (*page_response)(struct device *dev,
 			     struct iommu_fault_event *evt,
 			     struct iommu_page_response *msg);
 	int (*def_domain_type)(struct device *dev);
+	void (*remove_dev_pasid)(struct device *dev, ioasid_t pasid);
 	const struct iommu_domain_ops *default_domain_ops;
 	unsigned long pgsize_bitmap;
 /**
  * struct iommu_domain_ops - domain specific operations
  * @attach_dev: attach an iommu domain to a device
+ *  Return:
+ * * 0		- success
+ * * EINVAL	- can indicate that device and domain are incompatible due to
+ *		  some previous configuration of the domain, in which case the
+ *		  driver shouldn't log an error, since it is legitimate for a
+ *		  caller to test reuse of existing domains. Otherwise, it may
+ *		  still represent some other fundamental problem
+ * * ENOMEM	- out of memory
+ * * ENOSPC	- non-ENOMEM type of resource allocation failures
+ * * EBUSY	- device is attached to a domain and cannot be changed
+ * * ENODEV	- device specific errors, not able to be attached
+ * * <others>	- treated as ENODEV by the caller. Use is discouraged
  * @detach_dev: detach an iommu domain from a device
+ * @set_dev_pasid: set an iommu domain to a pasid of device
  * @map: map a physically contiguous memory region to an iommu domain
  * @map_pages: map a physically contiguous set of pages of the same size to
  *             an iommu domain.
 struct iommu_domain_ops {
 	int (*attach_dev)(struct iommu_domain *domain, struct device *dev);
 	void (*detach_dev)(struct iommu_domain *domain, struct device *dev);
+	int (*set_dev_pasid)(struct iommu_domain *domain, struct device *dev,
+			     ioasid_t pasid);
 	int (*map)(struct iommu_domain *domain, unsigned long iova,
 		   phys_addr_t paddr, size_t size, int prot, gfp_t gfp);
  * @list: Used by the iommu-core to keep a list of registered iommus
  * @ops: iommu-ops for talking to this iommu
  * @dev: struct device for sysfs handling
+ * @max_pasids: number of supported PASIDs
  */
 struct iommu_device {
 	struct list_head list;
 	const struct iommu_ops *ops;
 	struct fwnode_handle *fwnode;
 	struct device *dev;
+	u32 max_pasids;
 };
 /**
  * @fwspec:	 IOMMU fwspec data
  * @iommu_dev:	 IOMMU device this device is linked to
  * @priv:	 IOMMU Driver private data
+ * @max_pasids:  number of PASIDs this device can consume
  *
  * TODO: migrate other per device data pointers under iommu_dev_data, e.g.
  *	struct iommu_group	*iommu_group;
 	struct iommu_fwspec		*fwspec;
 	struct iommu_device		*iommu_dev;
 	void				*priv;
+	u32				max_pasids;
 };
 int iommu_device_register(struct iommu_device *iommu,
  */
 struct iommu_sva {
 	struct device			*dev;
+	struct iommu_domain		*domain;
 };
 int iommu_fwspec_init(struct device *dev, struct fwnode_handle *iommu_fwnode,
 int iommu_dev_enable_feature(struct device *dev, enum iommu_dev_features f);
 int iommu_dev_disable_feature(struct device *dev, enum iommu_dev_features f);
-struct iommu_sva *iommu_sva_bind_device(struct device *dev,
-					struct mm_struct *mm,
-					void *drvdata);
-void iommu_sva_unbind_device(struct iommu_sva *handle);
-u32 iommu_sva_get_pasid(struct iommu_sva *handle);
 int iommu_device_use_default_domain(struct device *dev);
 void iommu_device_unuse_default_domain(struct device *dev);
 void iommu_group_release_dma_owner(struct iommu_group *group);
 bool iommu_group_dma_owner_claimed(struct iommu_group *group);
+int iommu_device_claim_dma_owner(struct device *dev, void *owner);
+void iommu_device_release_dma_owner(struct device *dev);
+struct iommu_domain *iommu_sva_domain_alloc(struct device *dev,
+					    struct mm_struct *mm);
+int iommu_attach_device_pasid(struct iommu_domain *domain,
+			      struct device *dev, ioasid_t pasid);
+void iommu_detach_device_pasid(struct iommu_domain *domain,
+			       struct device *dev, ioasid_t pasid);
+struct iommu_domain *
+iommu_get_domain_for_dev_pasid(struct device *dev, ioasid_t pasid,
+			       unsigned int type);
 #else /* CONFIG_IOMMU_API */
 struct iommu_ops {};
 	return -ENODEV;
 }
-static inline struct iommu_sva *
-iommu_sva_bind_device(struct device *dev, struct mm_struct *mm, void *drvdata)
-{
-	return NULL;
-}
-static inline void iommu_sva_unbind_device(struct iommu_sva *handle)
-{
-}
-static inline u32 iommu_sva_get_pasid(struct iommu_sva *handle)
-{
-	return IOMMU_PASID_INVALID;
-}
 static inline struct iommu_fwspec *dev_iommu_fwspec_get(struct device *dev)
 {
 	return NULL;
 {
 	return false;
 }
+static inline void iommu_device_release_dma_owner(struct device *dev)
+{
+}
+static inline int iommu_device_claim_dma_owner(struct device *dev, void *owner)
+{
+	return -ENODEV;
+}
+static inline struct iommu_domain *
+iommu_sva_domain_alloc(struct device *dev, struct mm_struct *mm)
+{
+	return NULL;
+}
+static inline int iommu_attach_device_pasid(struct iommu_domain *domain,
+					    struct device *dev, ioasid_t pasid)
+{
+	return -ENODEV;
+}
+static inline void iommu_detach_device_pasid(struct iommu_domain *domain,
+					     struct device *dev, ioasid_t pasid)
+{
+}
+static inline struct iommu_domain *
+iommu_get_domain_for_dev_pasid(struct device *dev, ioasid_t pasid,
+			       unsigned int type)
+{
+	return NULL;
+}
 #endif /* CONFIG_IOMMU_API */
 /**
 	return false;
 }
+#ifdef CONFIG_IOMMU_SVA
+struct iommu_sva *iommu_sva_bind_device(struct device *dev,
+					struct mm_struct *mm);
+void iommu_sva_unbind_device(struct iommu_sva *handle);
+u32 iommu_sva_get_pasid(struct iommu_sva *handle);
+#else
+static inline struct iommu_sva *
+iommu_sva_bind_device(struct device *dev, struct mm_struct *mm)
+{
+	return NULL;
+}
+static inline void iommu_sva_unbind_device(struct iommu_sva *handle)
+{
+}
+static inline u32 iommu_sva_get_pasid(struct iommu_sva *handle)
+{
+	return IOMMU_PASID_INVALID;
+}
+#endif /* CONFIG_IOMMU_SVA */
 #endif /* __LINUX_IOMMU_H */

include/linux/iommufd.h+98−0

@@ -0,0 +1,98 @@
	1	/* SPDX-License-Identifier: GPL-2.0-only */
	2	/*
	3	* Copyright (C) 2021 Intel Corporation
	4	* Copyright (c) 2021-2022, NVIDIA CORPORATION & AFFILIATES
	5	*/
	6	#ifndef __LINUX_IOMMUFD_H
	7	#define __LINUX_IOMMUFD_H
	8
	9	#include <linux/types.h>
	10	#include <linux/errno.h>
	11	#include <linux/err.h>
	12
	13	struct device;
	14	struct iommufd_device;
	15	struct page;
	16	struct iommufd_ctx;
	17	struct iommufd_access;
	18	struct file;
	19
	20	struct iommufd_device iommufd_device_bind(struct iommufd_ctx ictx,
	21	struct device dev, u32 id);
	22	void iommufd_device_unbind(struct iommufd_device *idev);
	23
	24	int iommufd_device_attach(struct iommufd_device idev, u32 pt_id);
	25	void iommufd_device_detach(struct iommufd_device *idev);
	26
	27	struct iommufd_access_ops {
	28	u8 needs_pin_pages : 1;
	29	void (unmap)(void data, unsigned long iova, unsigned long length);
	30	};
	31
	32	enum {
	33	IOMMUFD_ACCESS_RW_READ = 0,
	34	IOMMUFD_ACCESS_RW_WRITE = 1 << 0,
	35	/* Set if the caller is in a kthread then rw will use kthread_use_mm() */
	36	IOMMUFD_ACCESS_RW_KTHREAD = 1 << 1,
	37
	38	/* Only for use by selftest */
	39	__IOMMUFD_ACCESS_RW_SLOW_PATH = 1 << 2,
	40	};
	41
	42	struct iommufd_access *
	43	iommufd_access_create(struct iommufd_ctx *ictx, u32 ioas_id,
	44	const struct iommufd_access_ops ops, void data);
	45	void iommufd_access_destroy(struct iommufd_access *access);
	46
	47	void iommufd_ctx_get(struct iommufd_ctx *ictx);
	48
	49	#if IS_ENABLED(CONFIG_IOMMUFD)
	50	struct iommufd_ctx iommufd_ctx_from_file(struct file file);
	51	void iommufd_ctx_put(struct iommufd_ctx *ictx);
	52
	53	int iommufd_access_pin_pages(struct iommufd_access *access, unsigned long iova,
	54	unsigned long length, struct page **out_pages,
	55	unsigned int flags);
	56	void iommufd_access_unpin_pages(struct iommufd_access *access,
	57	unsigned long iova, unsigned long length);
	58	int iommufd_access_rw(struct iommufd_access *access, unsigned long iova,
	59	void *data, size_t len, unsigned int flags);
	60	int iommufd_vfio_compat_ioas_id(struct iommufd_ctx ictx, u32 out_ioas_id);
	61	#else /* !CONFIG_IOMMUFD */
	62	static inline struct iommufd_ctx iommufd_ctx_from_file(struct file file)
	63	{
	64	return ERR_PTR(-EOPNOTSUPP);
	65	}
	66
	67	static inline void iommufd_ctx_put(struct iommufd_ctx *ictx)
	68	{
	69	}
	70
	71	static inline int iommufd_access_pin_pages(struct iommufd_access *access,
	72	unsigned long iova,
	73	unsigned long length,
	74	struct page **out_pages,
	75	unsigned int flags)
	76	{
	77	return -EOPNOTSUPP;
	78	}
	79
	80	static inline void iommufd_access_unpin_pages(struct iommufd_access *access,
	81	unsigned long iova,
	82	unsigned long length)
	83	{
	84	}
	85
	86	static inline int iommufd_access_rw(struct iommufd_access *access, unsigned long iova,
	87	void *data, size_t len, unsigned int flags)
	88	{
	89	return -EOPNOTSUPP;
	90	}
	91
	92	static inline int iommufd_vfio_compat_ioas_id(struct iommufd_ctx *ictx,
	93	u32 *out_ioas_id)
	94	{
	95	return -EOPNOTSUPP;
	96	}
	97	#endif /* CONFIG_IOMMUFD */
	98	#endif

include/linux/sched/user.h+1−1

@@ -25,7 +25,7 @@ struct user_struct { · 24 unmodified lines
25		25
26	#if defined(CONFIG_PERF_EVENTS) \|\| defined(CONFIG_BPF_SYSCALL) \|\| \	26	#if defined(CONFIG_PERF_EVENTS) \|\| defined(CONFIG_BPF_SYSCALL) \|\| \
27	defined(CONFIG_NET) \|\| defined(CONFIG_IO_URING) \|\| \	27	defined(CONFIG_NET) \|\| defined(CONFIG_IO_URING) \|\| \
28	defined(CONFIG_VFIO_PCI_ZDEV_KVM)	28	defined(CONFIG_VFIO_PCI_ZDEV_KVM) \|\| IS_ENABLED(CONFIG_IOMMUFD)
29	atomic_long_t locked_vm;	29	atomic_long_t locked_vm;
30	#endif	30	#endif
31	#ifdef CONFIG_WATCH_QUEUE	31	#ifdef CONFIG_WATCH_QUEUE

include/linux/vfio.h+39−0

@@ -17,6 +17,9 @@ · 16 unmodified lines
17	#include <linux/iova_bitmap.h>	17	#include <linux/iova_bitmap.h>
18		18
19	struct kvm;	19	struct kvm;
		20	struct iommufd_ctx;
		21	struct iommufd_device;
		22	struct iommufd_access;
20		23
21	/*	24	/*
22	* VFIO devices can be placed in a set, this allows all devices to share this	25	* VFIO devices can be placed in a set, this allows all devices to share this
@@ -54,6 +57,12 @@ struct vfio_device { · 31 unmodified lines
54	struct completion comp;	57	struct completion comp;
55	struct list_head group_next;	58	struct list_head group_next;
56	struct list_head iommu_entry;	59	struct list_head iommu_entry;
		60	struct iommufd_access *iommufd_access;
		61	#if IS_ENABLED(CONFIG_IOMMUFD)
		62	struct iommufd_device *iommufd_device;
		63	struct iommufd_ctx *iommufd_ictx;
		64	bool iommufd_attached;
		65	#endif
57	};	66	};
58		67
59	/**	68	/**
@@ -80,6 +89,10 @@ struct vfio_device_ops { · 20 unmodified lines
80	char *name;	89	char *name;
81	int (init)(struct vfio_device vdev);	90	int (init)(struct vfio_device vdev);
82	void (release)(struct vfio_device vdev);	91	void (release)(struct vfio_device vdev);
		92	int (bind_iommufd)(struct vfio_device vdev,
		93	struct iommufd_ctx ictx, u32 out_device_id);
		94	void (unbind_iommufd)(struct vfio_device vdev);
		95	int (attach_ioas)(struct vfio_device vdev, u32 *pt_id);
83	int (open_device)(struct vfio_device vdev);	96	int (open_device)(struct vfio_device vdev);
84	void (close_device)(struct vfio_device vdev);	97	void (close_device)(struct vfio_device vdev);
85	ssize_t (read)(struct vfio_device vdev, char __user *buf,	98	ssize_t (read)(struct vfio_device vdev, char __user *buf,
@@ -96,6 +109,32 @@ struct vfio_device_ops { · 10 unmodified lines
96	void __user *arg, size_t argsz);	109	void __user *arg, size_t argsz);
97	};	110	};
98		111
		112	#if IS_ENABLED(CONFIG_IOMMUFD)
		113	int vfio_iommufd_physical_bind(struct vfio_device *vdev,
		114	struct iommufd_ctx ictx, u32 out_device_id);
		115	void vfio_iommufd_physical_unbind(struct vfio_device *vdev);
		116	int vfio_iommufd_physical_attach_ioas(struct vfio_device vdev, u32 pt_id);
		117	int vfio_iommufd_emulated_bind(struct vfio_device *vdev,
		118	struct iommufd_ctx ictx, u32 out_device_id);
		119	void vfio_iommufd_emulated_unbind(struct vfio_device *vdev);
		120	int vfio_iommufd_emulated_attach_ioas(struct vfio_device vdev, u32 pt_id);
		121	#else
		122	#define vfio_iommufd_physical_bind \
		123	((int ()(struct vfio_device vdev, struct iommufd_ctx *ictx, \
		124	u32 *out_device_id)) NULL)
		125	#define vfio_iommufd_physical_unbind \
		126	((void ()(struct vfio_device vdev)) NULL)
		127	#define vfio_iommufd_physical_attach_ioas \
		128	((int ()(struct vfio_device vdev, u32 *pt_id)) NULL)
		129	#define vfio_iommufd_emulated_bind \
		130	((int ()(struct vfio_device vdev, struct iommufd_ctx *ictx, \
		131	u32 *out_device_id)) NULL)
		132	#define vfio_iommufd_emulated_unbind \
		133	((void ()(struct vfio_device vdev)) NULL)
		134	#define vfio_iommufd_emulated_attach_ioas \
		135	((int ()(struct vfio_device vdev, u32 *pt_id)) NULL)
		136	#endif
		137
99	/**	138	/**
100	* @migration_set_state: Optional callback to change the migration state for	139	* @migration_set_state: Optional callback to change the migration state for
101	* devices that support migration. It's mandatory for	140	* devices that support migration. It's mandatory for

include/uapi/linux/iommufd.h+347−0

@@ -0,0 +1,347 @@
	1	/* SPDX-License-Identifier: GPL-2.0 WITH Linux-syscall-note */
	2	/* Copyright (c) 2021-2022, NVIDIA CORPORATION & AFFILIATES.
	3	*/
	4	#ifndef _UAPI_IOMMUFD_H
	5	#define _UAPI_IOMMUFD_H
	6
	7	#include <linux/types.h>
	8	#include <linux/ioctl.h>
	9
	10	#define IOMMUFD_TYPE (';')
	11
	12	/**
	13	* DOC: General ioctl format
	14	*
	15	* The ioctl interface follows a general format to allow for extensibility. Each
	16	* ioctl is passed in a structure pointer as the argument providing the size of
	17	* the structure in the first u32. The kernel checks that any structure space
	18	* beyond what it understands is 0. This allows userspace to use the backward
	19	* compatible portion while consistently using the newer, larger, structures.
	20	*
	21	* ioctls use a standard meaning for common errnos:
	22	*
	23	* - ENOTTY: The IOCTL number itself is not supported at all
	24	* - E2BIG: The IOCTL number is supported, but the provided structure has
	25	* non-zero in a part the kernel does not understand.
	26	* - EOPNOTSUPP: The IOCTL number is supported, and the structure is
	27	* understood, however a known field has a value the kernel does not
	28	* understand or support.
	29	* - EINVAL: Everything about the IOCTL was understood, but a field is not
	30	* correct.
	31	* - ENOENT: An ID or IOVA provided does not exist.
	32	* - ENOMEM: Out of memory.
	33	* - EOVERFLOW: Mathematics overflowed.
	34	*
	35	* As well as additional errnos, within specific ioctls.
	36	*/
	37	enum {
	38	IOMMUFD_CMD_BASE = 0x80,
	39	IOMMUFD_CMD_DESTROY = IOMMUFD_CMD_BASE,
	40	IOMMUFD_CMD_IOAS_ALLOC,
	41	IOMMUFD_CMD_IOAS_ALLOW_IOVAS,
	42	IOMMUFD_CMD_IOAS_COPY,
	43	IOMMUFD_CMD_IOAS_IOVA_RANGES,
	44	IOMMUFD_CMD_IOAS_MAP,
	45	IOMMUFD_CMD_IOAS_UNMAP,
	46	IOMMUFD_CMD_OPTION,
	47	IOMMUFD_CMD_VFIO_IOAS,
	48	};
	49
	50	/**
	51	* struct iommu_destroy - ioctl(IOMMU_DESTROY)
	52	* @size: sizeof(struct iommu_destroy)
	53	* @id: iommufd object ID to destroy. Can be any destroyable object type.
	54	*
	55	* Destroy any object held within iommufd.
	56	*/
	57	struct iommu_destroy {
	58	__u32 size;
	59	__u32 id;
	60	};
	61	#define IOMMU_DESTROY _IO(IOMMUFD_TYPE, IOMMUFD_CMD_DESTROY)
	62
	63	/**
	64	* struct iommu_ioas_alloc - ioctl(IOMMU_IOAS_ALLOC)
	65	* @size: sizeof(struct iommu_ioas_alloc)
	66	* @flags: Must be 0
	67	* @out_ioas_id: Output IOAS ID for the allocated object
	68	*
	69	* Allocate an IO Address Space (IOAS) which holds an IO Virtual Address (IOVA)
	70	* to memory mapping.
	71	*/
	72	struct iommu_ioas_alloc {
	73	__u32 size;
	74	__u32 flags;
	75	__u32 out_ioas_id;
	76	};
	77	#define IOMMU_IOAS_ALLOC _IO(IOMMUFD_TYPE, IOMMUFD_CMD_IOAS_ALLOC)
	78
	79	/**
	80	* struct iommu_iova_range - ioctl(IOMMU_IOVA_RANGE)
	81	* @start: First IOVA
	82	* @last: Inclusive last IOVA
	83	*
	84	* An interval in IOVA space.
	85	*/
	86	struct iommu_iova_range {
	87	__aligned_u64 start;
	88	__aligned_u64 last;
	89	};
	90
	91	/**
	92	* struct iommu_ioas_iova_ranges - ioctl(IOMMU_IOAS_IOVA_RANGES)
	93	* @size: sizeof(struct iommu_ioas_iova_ranges)
	94	* @ioas_id: IOAS ID to read ranges from
	95	* @num_iovas: Input/Output total number of ranges in the IOAS
	96	* @__reserved: Must be 0
	97	* @allowed_iovas: Pointer to the output array of struct iommu_iova_range
	98	* @out_iova_alignment: Minimum alignment required for mapping IOVA
	99	*
	100	* Query an IOAS for ranges of allowed IOVAs. Mapping IOVA outside these ranges
	101	* is not allowed. num_iovas will be set to the total number of iovas and
	102	* the allowed_iovas[] will be filled in as space permits.
	103	*
	104	* The allowed ranges are dependent on the HW path the DMA operation takes, and
	105	* can change during the lifetime of the IOAS. A fresh empty IOAS will have a
	106	* full range, and each attached device will narrow the ranges based on that
	107	* device's HW restrictions. Detaching a device can widen the ranges. Userspace
	108	* should query ranges after every attach/detach to know what IOVAs are valid
	109	* for mapping.
	110	*
	111	* On input num_iovas is the length of the allowed_iovas array. On output it is
	112	* the total number of iovas filled in. The ioctl will return -EMSGSIZE and set
	113	* num_iovas to the required value if num_iovas is too small. In this case the
	114	* caller should allocate a larger output array and re-issue the ioctl.
	115	*
	116	* out_iova_alignment returns the minimum IOVA alignment that can be given
	117	* to IOMMU_IOAS_MAP/COPY. IOVA's must satisfy::
	118	*
	119	* starting_iova % out_iova_alignment == 0
	120	* (starting_iova + length) % out_iova_alignment == 0
	121	*
	122	* out_iova_alignment can be 1 indicating any IOVA is allowed. It cannot
	123	* be higher than the system PAGE_SIZE.
	124	*/
	125	struct iommu_ioas_iova_ranges {
	126	__u32 size;
	127	__u32 ioas_id;
	128	__u32 num_iovas;
	129	__u32 __reserved;
	130	__aligned_u64 allowed_iovas;
	131	__aligned_u64 out_iova_alignment;
	132	};
	133	#define IOMMU_IOAS_IOVA_RANGES _IO(IOMMUFD_TYPE, IOMMUFD_CMD_IOAS_IOVA_RANGES)
	134
	135	/**
	136	* struct iommu_ioas_allow_iovas - ioctl(IOMMU_IOAS_ALLOW_IOVAS)
	137	* @size: sizeof(struct iommu_ioas_allow_iovas)
	138	* @ioas_id: IOAS ID to allow IOVAs from
	139	* @num_iovas: Input/Output total number of ranges in the IOAS
	140	* @__reserved: Must be 0
	141	* @allowed_iovas: Pointer to array of struct iommu_iova_range
	142	*
	143	* Ensure a range of IOVAs are always available for allocation. If this call
	144	* succeeds then IOMMU_IOAS_IOVA_RANGES will never return a list of IOVA ranges
	145	* that are narrower than the ranges provided here. This call will fail if
	146	* IOMMU_IOAS_IOVA_RANGES is currently narrower than the given ranges.
	147	*
	148	* When an IOAS is first created the IOVA_RANGES will be maximally sized, and as
	149	* devices are attached the IOVA will narrow based on the device restrictions.
	150	* When an allowed range is specified any narrowing will be refused, ie device
	151	* attachment can fail if the device requires limiting within the allowed range.
	152	*
	153	* Automatic IOVA allocation is also impacted by this call. MAP will only
	154	* allocate within the allowed IOVAs if they are present.
	155	*
	156	* This call replaces the entire allowed list with the given list.
	157	*/
	158	struct iommu_ioas_allow_iovas {
	159	__u32 size;
	160	__u32 ioas_id;
	161	__u32 num_iovas;
	162	__u32 __reserved;
	163	__aligned_u64 allowed_iovas;
	164	};
	165	#define IOMMU_IOAS_ALLOW_IOVAS _IO(IOMMUFD_TYPE, IOMMUFD_CMD_IOAS_ALLOW_IOVAS)
	166
	167	/**
	168	* enum iommufd_ioas_map_flags - Flags for map and copy
	169	* @IOMMU_IOAS_MAP_FIXED_IOVA: If clear the kernel will compute an appropriate
	170	* IOVA to place the mapping at
	171	* @IOMMU_IOAS_MAP_WRITEABLE: DMA is allowed to write to this mapping
	172	* @IOMMU_IOAS_MAP_READABLE: DMA is allowed to read from this mapping
	173	*/
	174	enum iommufd_ioas_map_flags {
	175	IOMMU_IOAS_MAP_FIXED_IOVA = 1 << 0,
	176	IOMMU_IOAS_MAP_WRITEABLE = 1 << 1,
	177	IOMMU_IOAS_MAP_READABLE = 1 << 2,
	178	};
	179
	180	/**
	181	* struct iommu_ioas_map - ioctl(IOMMU_IOAS_MAP)
	182	* @size: sizeof(struct iommu_ioas_map)
	183	* @flags: Combination of enum iommufd_ioas_map_flags
	184	* @ioas_id: IOAS ID to change the mapping of
	185	* @__reserved: Must be 0
	186	* @user_va: Userspace pointer to start mapping from
	187	* @length: Number of bytes to map
	188	* @iova: IOVA the mapping was placed at. If IOMMU_IOAS_MAP_FIXED_IOVA is set
	189	* then this must be provided as input.
	190	*
	191	* Set an IOVA mapping from a user pointer. If FIXED_IOVA is specified then the
	192	* mapping will be established at iova, otherwise a suitable location based on
	193	* the reserved and allowed lists will be automatically selected and returned in
	194	* iova.
	195	*
	196	* If IOMMU_IOAS_MAP_FIXED_IOVA is specified then the iova range must currently
	197	* be unused, existing IOVA cannot be replaced.
	198	*/
	199	struct iommu_ioas_map {
	200	__u32 size;
	201	__u32 flags;
	202	__u32 ioas_id;
	203	__u32 __reserved;
	204	__aligned_u64 user_va;
	205	__aligned_u64 length;
	206	__aligned_u64 iova;
	207	};
	208	#define IOMMU_IOAS_MAP _IO(IOMMUFD_TYPE, IOMMUFD_CMD_IOAS_MAP)
	209
	210	/**
	211	* struct iommu_ioas_copy - ioctl(IOMMU_IOAS_COPY)
	212	* @size: sizeof(struct iommu_ioas_copy)
	213	* @flags: Combination of enum iommufd_ioas_map_flags
	214	* @dst_ioas_id: IOAS ID to change the mapping of
	215	* @src_ioas_id: IOAS ID to copy from
	216	* @length: Number of bytes to copy and map
	217	* @dst_iova: IOVA the mapping was placed at. If IOMMU_IOAS_MAP_FIXED_IOVA is
	218	* set then this must be provided as input.
	219	* @src_iova: IOVA to start the copy
	220	*
	221	* Copy an already existing mapping from src_ioas_id and establish it in
	222	* dst_ioas_id. The src iova/length must exactly match a range used with
	223	* IOMMU_IOAS_MAP.
	224	*
	225	* This may be used to efficiently clone a subset of an IOAS to another, or as a
	226	* kind of 'cache' to speed up mapping. Copy has an efficiency advantage over
	227	* establishing equivalent new mappings, as internal resources are shared, and
	228	* the kernel will pin the user memory only once.
	229	*/
	230	struct iommu_ioas_copy {
	231	__u32 size;
	232	__u32 flags;
	233	__u32 dst_ioas_id;
	234	__u32 src_ioas_id;
	235	__aligned_u64 length;
	236	__aligned_u64 dst_iova;
	237	__aligned_u64 src_iova;
	238	};
	239	#define IOMMU_IOAS_COPY _IO(IOMMUFD_TYPE, IOMMUFD_CMD_IOAS_COPY)
	240
	241	/**
	242	* struct iommu_ioas_unmap - ioctl(IOMMU_IOAS_UNMAP)
	243	* @size: sizeof(struct iommu_ioas_unmap)
	244	* @ioas_id: IOAS ID to change the mapping of
	245	* @iova: IOVA to start the unmapping at
	246	* @length: Number of bytes to unmap, and return back the bytes unmapped
	247	*
	248	* Unmap an IOVA range. The iova/length must be a superset of a previously
	249	* mapped range used with IOMMU_IOAS_MAP or IOMMU_IOAS_COPY. Splitting or
	250	* truncating ranges is not allowed. The values 0 to U64_MAX will unmap
	251	* everything.
	252	*/
	253	struct iommu_ioas_unmap {
	254	__u32 size;
	255	__u32 ioas_id;
	256	__aligned_u64 iova;
	257	__aligned_u64 length;
	258	};
	259	#define IOMMU_IOAS_UNMAP _IO(IOMMUFD_TYPE, IOMMUFD_CMD_IOAS_UNMAP)
	260
	261	/**
	262	* enum iommufd_option - ioctl(IOMMU_OPTION_RLIMIT_MODE) and
	263	* ioctl(IOMMU_OPTION_HUGE_PAGES)
	264	* @IOMMU_OPTION_RLIMIT_MODE:
	265	* Change how RLIMIT_MEMLOCK accounting works. The caller must have privilege
	266	* to invoke this. Value 0 (default) is user based accouting, 1 uses process
	267	* based accounting. Global option, object_id must be 0
	268	* @IOMMU_OPTION_HUGE_PAGES:
	269	* Value 1 (default) allows contiguous pages to be combined when generating
	270	* iommu mappings. Value 0 disables combining, everything is mapped to
	271	* PAGE_SIZE. This can be useful for benchmarking. This is a per-IOAS
	272	* option, the object_id must be the IOAS ID.
	273	*/
	274	enum iommufd_option {
	275	IOMMU_OPTION_RLIMIT_MODE = 0,
	276	IOMMU_OPTION_HUGE_PAGES = 1,
	277	};
	278
	279	/**
	280	* enum iommufd_option_ops - ioctl(IOMMU_OPTION_OP_SET) and
	281	* ioctl(IOMMU_OPTION_OP_GET)
	282	* @IOMMU_OPTION_OP_SET: Set the option's value
	283	* @IOMMU_OPTION_OP_GET: Get the option's value
	284	*/
	285	enum iommufd_option_ops {
	286	IOMMU_OPTION_OP_SET = 0,
	287	IOMMU_OPTION_OP_GET = 1,
	288	};
	289
	290	/**
	291	* struct iommu_option - iommu option multiplexer
	292	* @size: sizeof(struct iommu_option)
	293	* @option_id: One of enum iommufd_option
	294	* @op: One of enum iommufd_option_ops
	295	* @__reserved: Must be 0
	296	* @object_id: ID of the object if required
	297	* @val64: Option value to set or value returned on get
	298	*
	299	* Change a simple option value. This multiplexor allows controlling options
	300	* on objects. IOMMU_OPTION_OP_SET will load an option and IOMMU_OPTION_OP_GET
	301	* will return the current value.
	302	*/
	303	struct iommu_option {
	304	__u32 size;
	305	__u32 option_id;
	306	__u16 op;
	307	__u16 __reserved;
	308	__u32 object_id;
	309	__aligned_u64 val64;
	310	};
	311	#define IOMMU_OPTION _IO(IOMMUFD_TYPE, IOMMUFD_CMD_OPTION)
	312
	313	/**
	314	* enum iommufd_vfio_ioas_op - IOMMU_VFIO_IOAS_* ioctls
	315	* @IOMMU_VFIO_IOAS_GET: Get the current compatibility IOAS
	316	* @IOMMU_VFIO_IOAS_SET: Change the current compatibility IOAS
	317	* @IOMMU_VFIO_IOAS_CLEAR: Disable VFIO compatibility
	318	*/
	319	enum iommufd_vfio_ioas_op {
	320	IOMMU_VFIO_IOAS_GET = 0,
	321	IOMMU_VFIO_IOAS_SET = 1,
	322	IOMMU_VFIO_IOAS_CLEAR = 2,
	323	};
	324
	325	/**
	326	* struct iommu_vfio_ioas - ioctl(IOMMU_VFIO_IOAS)
	327	* @size: sizeof(struct iommu_vfio_ioas)
	328	* @ioas_id: For IOMMU_VFIO_IOAS_SET the input IOAS ID to set
	329	* For IOMMU_VFIO_IOAS_GET will output the IOAS ID
	330	* @op: One of enum iommufd_vfio_ioas_op
	331	* @__reserved: Must be 0
	332	*
	333	* The VFIO compatibility support uses a single ioas because VFIO APIs do not
	334	* support the ID field. Set or Get the IOAS that VFIO compatibility will use.
	335	* When VFIO_GROUP_SET_CONTAINER is used on an iommufd it will get the
	336	* compatibility ioas, either by taking what is already set, or auto creating
	337	* one. From then on VFIO will continue to use that ioas and is not effected by
	338	* this ioctl. SET or CLEAR does not destroy any auto-created IOAS.
	339	*/
	340	struct iommu_vfio_ioas {
	341	__u32 size;
	342	__u32 ioas_id;
	343	__u16 op;
	344	__u16 __reserved;
	345	};
	346	#define IOMMU_VFIO_IOAS _IO(IOMMUFD_TYPE, IOMMUFD_CMD_VFIO_IOAS)
	347	#endif

kernel/user.c+1−0

@@ -185,6 +185,7 @@ void free_uid(struct user_struct *up) · 184 unmodified lines
185	if (refcount_dec_and_lock_irqsave(&up->__count, &uidhash_lock, &flags))	185	if (refcount_dec_and_lock_irqsave(&up->__count, &uidhash_lock, &flags))
186	free_user(up, flags);	186	free_user(up, flags);
187	}	187	}
		188	EXPORT_SYMBOL_GPL(free_uid);
188		189
189	struct user_struct *alloc_uid(kuid_t uid)	190	struct user_struct *alloc_uid(kuid_t uid)
190	{	191	{

lib/Kconfig+4−0

@@ -480,6 +480,10 @@ config INTERVAL_TREE · 479 unmodified lines
480		480
481	for more information.	481	for more information.
482		482
		483	config INTERVAL_TREE_SPAN_ITER
		484	bool
		485	depends on INTERVAL_TREE
		486
483	config XARRAY_MULTI	487	config XARRAY_MULTI
484	bool	488	bool
485	help	489	help

lib/interval_tree.c+132−0

@@ -15,3 +15,135 @@ EXPORT_SYMBOL_GPL(interval_tree_insert); · 14 unmodified lines
15	EXPORT_SYMBOL_GPL(interval_tree_remove);	15	EXPORT_SYMBOL_GPL(interval_tree_remove);
16	EXPORT_SYMBOL_GPL(interval_tree_iter_first);	16	EXPORT_SYMBOL_GPL(interval_tree_iter_first);
17	EXPORT_SYMBOL_GPL(interval_tree_iter_next);	17	EXPORT_SYMBOL_GPL(interval_tree_iter_next);
		18
		19	#ifdef CONFIG_INTERVAL_TREE_SPAN_ITER
		20	/*
		21	* Roll nodes[1] into nodes[0] by advancing nodes[1] to the end of a contiguous
		22	* span of nodes. This makes nodes[0]->last the end of that contiguous used span
		23	* indexes that started at the original nodes[1]->start. nodes[1] is now the
		24	* first node starting the next used span. A hole span is between nodes[0]->last
		25	* and nodes[1]->start. nodes[1] must be !NULL.
		26	*/
		27	static void
		28	interval_tree_span_iter_next_gap(struct interval_tree_span_iter *state)
		29	{
		30	struct interval_tree_node *cur = state->nodes[1];
		31
		32	state->nodes[0] = cur;
		33	do {
		34	if (cur->last > state->nodes[0]->last)
		35	state->nodes[0] = cur;
		36	cur = interval_tree_iter_next(cur, state->first_index,
		37	state->last_index);
		38	} while (cur && (state->nodes[0]->last >= cur->start \|\|
		39	state->nodes[0]->last + 1 == cur->start));
		40	state->nodes[1] = cur;
		41	}
		42
		43	void interval_tree_span_iter_first(struct interval_tree_span_iter *iter,
		44	struct rb_root_cached *itree,
		45	unsigned long first_index,
		46	unsigned long last_index)
		47	{
		48	iter->first_index = first_index;
		49	iter->last_index = last_index;
		50	iter->nodes[0] = NULL;
		51	iter->nodes[1] =
		52	interval_tree_iter_first(itree, first_index, last_index);
		53	if (!iter->nodes[1]) {
		54	/* No nodes intersect the span, whole span is hole */
		55	iter->start_hole = first_index;
		56	iter->last_hole = last_index;
		57	iter->is_hole = 1;
		58	return;
		59	}
		60	if (iter->nodes[1]->start > first_index) {
		61	/* Leading hole on first iteration */
		62	iter->start_hole = first_index;
		63	iter->last_hole = iter->nodes[1]->start - 1;
		64	iter->is_hole = 1;
		65	interval_tree_span_iter_next_gap(iter);
		66	return;
		67	}
		68
		69	/* Starting inside a used */
		70	iter->start_used = first_index;
		71	iter->is_hole = 0;
		72	interval_tree_span_iter_next_gap(iter);
		73	iter->last_used = iter->nodes[0]->last;
		74	if (iter->last_used >= last_index) {
		75	iter->last_used = last_index;
		76	iter->nodes[0] = NULL;
		77	iter->nodes[1] = NULL;
		78	}
		79	}
		80	EXPORT_SYMBOL_GPL(interval_tree_span_iter_first);
		81
		82	void interval_tree_span_iter_next(struct interval_tree_span_iter *iter)
		83	{
		84	if (!iter->nodes[0] && !iter->nodes[1]) {
		85	iter->is_hole = -1;
		86	return;
		87	}
		88
		89	if (iter->is_hole) {
		90	iter->start_used = iter->last_hole + 1;
		91	iter->last_used = iter->nodes[0]->last;
		92	if (iter->last_used >= iter->last_index) {
		93	iter->last_used = iter->last_index;
		94	iter->nodes[0] = NULL;
		95	iter->nodes[1] = NULL;
		96	}
		97	iter->is_hole = 0;
		98	return;
		99	}
		100
		101	if (!iter->nodes[1]) {
		102	/* Trailing hole */
		103	iter->start_hole = iter->nodes[0]->last + 1;
		104	iter->last_hole = iter->last_index;
		105	iter->nodes[0] = NULL;
		106	iter->is_hole = 1;
		107	return;
		108	}
		109
		110	/* must have both nodes[0] and [1], interior hole */
		111	iter->start_hole = iter->nodes[0]->last + 1;
		112	iter->last_hole = iter->nodes[1]->start - 1;
		113	iter->is_hole = 1;
		114	interval_tree_span_iter_next_gap(iter);
		115	}
		116	EXPORT_SYMBOL_GPL(interval_tree_span_iter_next);
		117
		118	/*
		119	* Advance the iterator index to a specific position. The returned used/hole is
		120	* updated to start at new_index. This is faster than calling
		121	* interval_tree_span_iter_first() as it can avoid full searches in several
		122	* cases where the iterator is already set.
		123	*/
		124	void interval_tree_span_iter_advance(struct interval_tree_span_iter *iter,
		125	struct rb_root_cached *itree,
		126	unsigned long new_index)
		127	{
		128	if (iter->is_hole == -1)
		129	return;
		130
		131	iter->first_index = new_index;
		132	if (new_index > iter->last_index) {
		133	iter->is_hole = -1;
		134	return;
		135	}
		136
		137	/* Rely on the union aliasing hole/used */
		138	if (iter->start_hole <= new_index && new_index <= iter->last_hole) {
		139	iter->start_hole = new_index;
		140	return;
		141	}
		142	if (new_index == iter->last_hole + 1)
		143	interval_tree_span_iter_next(iter);
		144	else
		145	interval_tree_span_iter_first(iter, itree, new_index,
		146	iter->last_index);
		147	}
		148	EXPORT_SYMBOL_GPL(interval_tree_span_iter_advance);
		149	#endif

scripts/kernel-doc+8−4

 my $doc_inline_end = '^\s*\*/\s*$';
 my $doc_inline_oneline = '^\s*/\*\*\s*(@[\w\s]+):\s*(.*)\s*\*/\s*$';
 my $export_symbol = '^\s*EXPORT_SYMBOL(_GPL)?\s*\(\s*(\w+)\s*\)\s*;';
+my $export_symbol_ns = '^\s*EXPORT_SYMBOL_NS(_GPL)?\s*\(\s*(\w+)\s*,\s*\w+\)\s*;';
 my $function_pointer = qr{([^\(]*\(\*)\s*\)\s*\(([^\)]*)\)};
 my $attribute = qr{__attribute__\s*\(\([a-z0-9,_\*\s\(\)]*\)\)}i;
 	    next if (defined($nosymbol_table{$2}));
 	    $function_table{$2} = 1;
 	}
+	if (/$export_symbol_ns/) {
+	    next if (defined($nosymbol_table{$2}));
+	    $function_table{$2} = 1;
+	}
     }
     close(IN);
 =item -export
 Only output documentation for the symbols that have been exported using
-EXPORT_SYMBOL() or EXPORT_SYMBOL_GPL() in any input FILE or -export-file FILE.
+EXPORT_SYMBOL() and related macros in any input FILE or -export-file FILE.
 =item -internal
 Only output documentation for the symbols that have NOT been exported using
-EXPORT_SYMBOL() or EXPORT_SYMBOL_GPL() in any input FILE or -export-file FILE.
+EXPORT_SYMBOL() and related macros in any input FILE or -export-file FILE.
 =item -function NAME
 =item -export-file FILE
-Specify an additional FILE in which to look for EXPORT_SYMBOL() and
+Specify an additional FILE in which to look for EXPORT_SYMBOL information.
-EXPORT_SYMBOL_GPL().
 To be used with -export or -internal.

tools/testing/selftests/Makefile+1−0

@@ -27,6 +27,7 @@ TARGETS += ftrace · 26 unmodified lines
27	TARGETS += futex	27	TARGETS += futex
28	TARGETS += gpio	28	TARGETS += gpio
29	TARGETS += intel_pstate	29	TARGETS += intel_pstate
		30	TARGETS += iommu
30	TARGETS += ipc	31	TARGETS += ipc
31	TARGETS += ir	32	TARGETS += ir
32	TARGETS += kcmp	33	TARGETS += kcmp

tools/testing/selftests/iommu/.gitignore+3−0

@@ -0,0 +1,3 @@
	1	# SPDX-License-Identifier: GPL-2.0-only
	2	/iommufd
	3	/iommufd_fail_nth

tools/testing/selftests/iommu/Makefile+12−0

@@ -0,0 +1,12 @@
	1	# SPDX-License-Identifier: GPL-2.0-only
	2	CFLAGS += -Wall -O2 -Wno-unused-function
	3	CFLAGS += -I../../../../include/uapi/
	4	CFLAGS += -I../../../../include/
	5
	6	CFLAGS += -D_GNU_SOURCE
	7
	8	TEST_GEN_PROGS :=
	9	TEST_GEN_PROGS += iommufd
	10	TEST_GEN_PROGS += iommufd_fail_nth
	11
	12	include ../lib.mk

tools/testing/selftests/iommu/config+2−0

@@ -0,0 +1,2 @@
	1	CONFIG_IOMMUFD
	2	CONFIG_IOMMUFD_TEST

tools/testing/selftests/iommu/iommufd.c+1654−0

@@ -0,0 +1,1654 @@
	1	// SPDX-License-Identifier: GPL-2.0-only
	2	/* Copyright (c) 2021-2022, NVIDIA CORPORATION & AFFILIATES */
	3	#include <stdlib.h>
	4	#include <sys/mman.h>
	5	#include <sys/eventfd.h>
	6
	7	#define __EXPORTED_HEADERS__
	8	#include <linux/vfio.h>
	9
	10	#include "iommufd_utils.h"
	11
	12	static void *buffer;
	13
	14	static unsigned long PAGE_SIZE;
	15	static unsigned long HUGEPAGE_SIZE;
	16
	17	#define MOCK_PAGE_SIZE (PAGE_SIZE / 2)
	18
	19	static unsigned long get_huge_page_size(void)
	20	{
	21	char buf[80];
	22	int ret;
	23	int fd;
	24
	25	fd = open("/sys/kernel/mm/transparent_hugepage/hpage_pmd_size",
	26	O_RDONLY);
	27	if (fd < 0)
	28	return 2 * 1024 * 1024;
	29
	30	ret = read(fd, buf, sizeof(buf));
	31	close(fd);
	32	if (ret <= 0 \|\| ret == sizeof(buf))
	33	return 2 * 1024 * 1024;
	34	buf[ret] = 0;
	35	return strtoul(buf, NULL, 10);
	36	}
	37
	38	static __attribute__((constructor)) void setup_sizes(void)
	39	{
	40	void *vrc;
	41	int rc;
	42
	43	PAGE_SIZE = sysconf(_SC_PAGE_SIZE);
	44	HUGEPAGE_SIZE = get_huge_page_size();
	45
	46	BUFFER_SIZE = PAGE_SIZE * 16;
	47	rc = posix_memalign(&buffer, HUGEPAGE_SIZE, BUFFER_SIZE);
	48	assert(!rc);
	49	assert(buffer);
	50	assert((uintptr_t)buffer % HUGEPAGE_SIZE == 0);
	51	vrc = mmap(buffer, BUFFER_SIZE, PROT_READ \| PROT_WRITE,
	52	MAP_SHARED \| MAP_ANONYMOUS \| MAP_FIXED, -1, 0);
	53	assert(vrc == buffer);
	54	}
	55
	56	FIXTURE(iommufd)
	57	{
	58	int fd;
	59	};
	60
	61	FIXTURE_SETUP(iommufd)
	62	{
	63	self->fd = open("/dev/iommu", O_RDWR);
	64	ASSERT_NE(-1, self->fd);
	65	}
	66
	67	FIXTURE_TEARDOWN(iommufd)
	68	{
	69	teardown_iommufd(self->fd, _metadata);
	70	}
	71
	72	TEST_F(iommufd, simple_close)
	73	{
	74	}
	75
	76	TEST_F(iommufd, cmd_fail)
	77	{
	78	struct iommu_destroy cmd = { .size = sizeof(cmd), .id = 0 };
	79
	80	/* object id is invalid */
	81	EXPECT_ERRNO(ENOENT, _test_ioctl_destroy(self->fd, 0));
	82	/* Bad pointer */
	83	EXPECT_ERRNO(EFAULT, ioctl(self->fd, IOMMU_DESTROY, NULL));
	84	/* Unknown ioctl */
	85	EXPECT_ERRNO(ENOTTY,
	86	ioctl(self->fd, _IO(IOMMUFD_TYPE, IOMMUFD_CMD_BASE - 1),
	87	&cmd));
	88	}
	89
	90	TEST_F(iommufd, cmd_length)
	91	{
	92	#define TEST_LENGTH(_struct, _ioctl) \
	93	{ \
	94	struct { \
	95	struct _struct cmd; \
	96	uint8_t extra; \
	97	} cmd = { .cmd = { .size = sizeof(struct _struct) - 1 }, \
	98	.extra = UINT8_MAX }; \
	99	int old_errno; \
	100	int rc; \
	101	\
	102	EXPECT_ERRNO(EINVAL, ioctl(self->fd, _ioctl, &cmd)); \
	103	cmd.cmd.size = sizeof(struct _struct) + 1; \
	104	EXPECT_ERRNO(E2BIG, ioctl(self->fd, _ioctl, &cmd)); \
	105	cmd.cmd.size = sizeof(struct _struct); \
	106	rc = ioctl(self->fd, _ioctl, &cmd); \
	107	old_errno = errno; \
	108	cmd.cmd.size = sizeof(struct _struct) + 1; \
	109	cmd.extra = 0; \
	110	if (rc) { \
	111	EXPECT_ERRNO(old_errno, \
	112	ioctl(self->fd, _ioctl, &cmd)); \
	113	} else { \
	114	ASSERT_EQ(0, ioctl(self->fd, _ioctl, &cmd)); \
	115	} \
	116	}
	117
	118	TEST_LENGTH(iommu_destroy, IOMMU_DESTROY);
	119	TEST_LENGTH(iommu_ioas_alloc, IOMMU_IOAS_ALLOC);
	120	TEST_LENGTH(iommu_ioas_iova_ranges, IOMMU_IOAS_IOVA_RANGES);
	121	TEST_LENGTH(iommu_ioas_allow_iovas, IOMMU_IOAS_ALLOW_IOVAS);
	122	TEST_LENGTH(iommu_ioas_map, IOMMU_IOAS_MAP);
	123	TEST_LENGTH(iommu_ioas_copy, IOMMU_IOAS_COPY);
	124	TEST_LENGTH(iommu_ioas_unmap, IOMMU_IOAS_UNMAP);
	125	TEST_LENGTH(iommu_option, IOMMU_OPTION);
	126	TEST_LENGTH(iommu_vfio_ioas, IOMMU_VFIO_IOAS);
	127	#undef TEST_LENGTH
	128	}
	129
	130	TEST_F(iommufd, cmd_ex_fail)
	131	{
	132	struct {
	133	struct iommu_destroy cmd;
	134	__u64 future;
	135	} cmd = { .cmd = { .size = sizeof(cmd), .id = 0 } };
	136
	137	/* object id is invalid and command is longer */
	138	EXPECT_ERRNO(ENOENT, ioctl(self->fd, IOMMU_DESTROY, &cmd));
	139	/* future area is non-zero */
	140	cmd.future = 1;
	141	EXPECT_ERRNO(E2BIG, ioctl(self->fd, IOMMU_DESTROY, &cmd));
	142	/* Original command "works" */
	143	cmd.cmd.size = sizeof(cmd.cmd);
	144	EXPECT_ERRNO(ENOENT, ioctl(self->fd, IOMMU_DESTROY, &cmd));
	145	/* Short command fails */
	146	cmd.cmd.size = sizeof(cmd.cmd) - 1;
	147	EXPECT_ERRNO(EINVAL, ioctl(self->fd, IOMMU_DESTROY, &cmd));
	148	}
	149
	150	TEST_F(iommufd, global_options)
	151	{
	152	struct iommu_option cmd = {
	153	.size = sizeof(cmd),
	154	.option_id = IOMMU_OPTION_RLIMIT_MODE,
	155	.op = IOMMU_OPTION_OP_GET,
	156	.val64 = 1,
	157	};
	158
	159	cmd.option_id = IOMMU_OPTION_RLIMIT_MODE;
	160	ASSERT_EQ(0, ioctl(self->fd, IOMMU_OPTION, &cmd));
	161	ASSERT_EQ(0, cmd.val64);
	162
	163	/* This requires root */
	164	cmd.op = IOMMU_OPTION_OP_SET;
	165	cmd.val64 = 1;
	166	ASSERT_EQ(0, ioctl(self->fd, IOMMU_OPTION, &cmd));
	167	cmd.val64 = 2;
	168	EXPECT_ERRNO(EINVAL, ioctl(self->fd, IOMMU_OPTION, &cmd));
	169
	170	cmd.op = IOMMU_OPTION_OP_GET;
	171	ASSERT_EQ(0, ioctl(self->fd, IOMMU_OPTION, &cmd));
	172	ASSERT_EQ(1, cmd.val64);
	173
	174	cmd.op = IOMMU_OPTION_OP_SET;
	175	cmd.val64 = 0;
	176	ASSERT_EQ(0, ioctl(self->fd, IOMMU_OPTION, &cmd));
	177
	178	cmd.op = IOMMU_OPTION_OP_GET;
	179	cmd.option_id = IOMMU_OPTION_HUGE_PAGES;
	180	EXPECT_ERRNO(ENOENT, ioctl(self->fd, IOMMU_OPTION, &cmd));
	181	cmd.op = IOMMU_OPTION_OP_SET;
	182	EXPECT_ERRNO(ENOENT, ioctl(self->fd, IOMMU_OPTION, &cmd));
	183	}
	184
	185	FIXTURE(iommufd_ioas)
	186	{
	187	int fd;
	188	uint32_t ioas_id;
	189	uint32_t domain_id;
	190	uint64_t base_iova;
	191	};
	192
	193	FIXTURE_VARIANT(iommufd_ioas)
	194	{
	195	unsigned int mock_domains;
	196	unsigned int memory_limit;
	197	};
	198
	199	FIXTURE_SETUP(iommufd_ioas)
	200	{
	201	unsigned int i;
	202
	203
	204	self->fd = open("/dev/iommu", O_RDWR);
	205	ASSERT_NE(-1, self->fd);
	206	test_ioctl_ioas_alloc(&self->ioas_id);
	207
	208	if (!variant->memory_limit) {
	209	test_ioctl_set_default_memory_limit();
	210	} else {
	211	test_ioctl_set_temp_memory_limit(variant->memory_limit);
	212	}
	213
	214	for (i = 0; i != variant->mock_domains; i++) {
	215	test_cmd_mock_domain(self->ioas_id, NULL, &self->domain_id);
	216	self->base_iova = MOCK_APERTURE_START;
	217	}
	218	}
	219
	220	FIXTURE_TEARDOWN(iommufd_ioas)
	221	{
	222	test_ioctl_set_default_memory_limit();
	223	teardown_iommufd(self->fd, _metadata);
	224	}
	225
	226	FIXTURE_VARIANT_ADD(iommufd_ioas, no_domain)
	227	{
	228	};
	229
	230	FIXTURE_VARIANT_ADD(iommufd_ioas, mock_domain)
	231	{
	232	.mock_domains = 1,
	233	};
	234
	235	FIXTURE_VARIANT_ADD(iommufd_ioas, two_mock_domain)
	236	{
	237	.mock_domains = 2,
	238	};
	239
	240	FIXTURE_VARIANT_ADD(iommufd_ioas, mock_domain_limit)
	241	{
	242	.mock_domains = 1,
	243	.memory_limit = 16,
	244	};
	245
	246	TEST_F(iommufd_ioas, ioas_auto_destroy)
	247	{
	248	}
	249
	250	TEST_F(iommufd_ioas, ioas_destroy)
	251	{
	252	if (self->domain_id) {
	253	/* IOAS cannot be freed while a domain is on it */
	254	EXPECT_ERRNO(EBUSY,
	255	_test_ioctl_destroy(self->fd, self->ioas_id));
	256	} else {
	257	/* Can allocate and manually free an IOAS table */
	258	test_ioctl_destroy(self->ioas_id);
	259	}
	260	}
	261
	262	TEST_F(iommufd_ioas, ioas_area_destroy)
	263	{
	264	/* Adding an area does not change ability to destroy */
	265	test_ioctl_ioas_map_fixed(buffer, PAGE_SIZE, self->base_iova);
	266	if (self->domain_id)
	267	EXPECT_ERRNO(EBUSY,
	268	_test_ioctl_destroy(self->fd, self->ioas_id));
	269	else
	270	test_ioctl_destroy(self->ioas_id);
	271	}
	272
	273	TEST_F(iommufd_ioas, ioas_area_auto_destroy)
	274	{
	275	int i;
	276
	277	/* Can allocate and automatically free an IOAS table with many areas */
	278	for (i = 0; i != 10; i++) {
	279	test_ioctl_ioas_map_fixed(buffer, PAGE_SIZE,
	280	self->base_iova + i * PAGE_SIZE);
	281	}
	282	}
	283
	284	TEST_F(iommufd_ioas, area)
	285	{
	286	int i;
	287
	288	/* Unmap fails if nothing is mapped */
	289	for (i = 0; i != 10; i++)
	290	test_err_ioctl_ioas_unmap(ENOENT, i * PAGE_SIZE, PAGE_SIZE);
	291
	292	/* Unmap works */
	293	for (i = 0; i != 10; i++)
	294	test_ioctl_ioas_map_fixed(buffer, PAGE_SIZE,
	295	self->base_iova + i * PAGE_SIZE);
	296	for (i = 0; i != 10; i++)
	297	test_ioctl_ioas_unmap(self->base_iova + i * PAGE_SIZE,
	298	PAGE_SIZE);
	299
	300	/* Split fails */
	301	test_ioctl_ioas_map_fixed(buffer, PAGE_SIZE * 2,
	302	self->base_iova + 16 * PAGE_SIZE);
	303	test_err_ioctl_ioas_unmap(ENOENT, self->base_iova + 16 * PAGE_SIZE,
	304	PAGE_SIZE);
	305	test_err_ioctl_ioas_unmap(ENOENT, self->base_iova + 17 * PAGE_SIZE,
	306	PAGE_SIZE);
	307
	308	/* Over map fails */
	309	test_err_ioctl_ioas_map_fixed(EEXIST, buffer, PAGE_SIZE * 2,
	310	self->base_iova + 16 * PAGE_SIZE);
	311	test_err_ioctl_ioas_map_fixed(EEXIST, buffer, PAGE_SIZE,
	312	self->base_iova + 16 * PAGE_SIZE);
	313	test_err_ioctl_ioas_map_fixed(EEXIST, buffer, PAGE_SIZE,
	314	self->base_iova + 17 * PAGE_SIZE);
	315	test_err_ioctl_ioas_map_fixed(EEXIST, buffer, PAGE_SIZE * 2,
	316	self->base_iova + 15 * PAGE_SIZE);
	317	test_err_ioctl_ioas_map_fixed(EEXIST, buffer, PAGE_SIZE * 3,
	318	self->base_iova + 15 * PAGE_SIZE);
	319
	320	/* unmap all works */
	321	test_ioctl_ioas_unmap(0, UINT64_MAX);
	322
	323	/* Unmap all succeeds on an empty IOAS */
	324	test_ioctl_ioas_unmap(0, UINT64_MAX);
	325	}
	326
	327	TEST_F(iommufd_ioas, unmap_fully_contained_areas)
	328	{
	329	uint64_t unmap_len;
	330	int i;
	331
	332	/* Give no_domain some space to rewind base_iova */
	333	self->base_iova += 4 * PAGE_SIZE;
	334
	335	for (i = 0; i != 4; i++)
	336	test_ioctl_ioas_map_fixed(buffer, 8 * PAGE_SIZE,
	337	self->base_iova + i * 16 * PAGE_SIZE);
	338
	339	/* Unmap not fully contained area doesn't work */
	340	test_err_ioctl_ioas_unmap(ENOENT, self->base_iova - 4 * PAGE_SIZE,
	341	8 * PAGE_SIZE);
	342	test_err_ioctl_ioas_unmap(ENOENT,
	343	self->base_iova + 3 * 16 * PAGE_SIZE +
	344	8 * PAGE_SIZE - 4 * PAGE_SIZE,
	345	8 * PAGE_SIZE);
	346
	347	/* Unmap fully contained areas works */
	348	ASSERT_EQ(0, _test_ioctl_ioas_unmap(self->fd, self->ioas_id,
	349	self->base_iova - 4 * PAGE_SIZE,
	350	3 * 16 * PAGE_SIZE + 8 * PAGE_SIZE +
	351	4 * PAGE_SIZE,
	352	&unmap_len));
	353	ASSERT_EQ(32 * PAGE_SIZE, unmap_len);
	354	}
	355
	356	TEST_F(iommufd_ioas, area_auto_iova)
	357	{
	358	struct iommu_test_cmd test_cmd = {
	359	.size = sizeof(test_cmd),
	360	.op = IOMMU_TEST_OP_ADD_RESERVED,
	361	.id = self->ioas_id,
	362	.add_reserved = { .start = PAGE_SIZE * 4,
	363	.length = PAGE_SIZE * 100 },
	364	};
	365	struct iommu_iova_range ranges[1] = {};
	366	struct iommu_ioas_allow_iovas allow_cmd = {
	367	.size = sizeof(allow_cmd),
	368	.ioas_id = self->ioas_id,
	369	.num_iovas = 1,
	370	.allowed_iovas = (uintptr_t)ranges,
	371	};
	372	__u64 iovas[10];
	373	int i;
	374
	375	/* Simple 4k pages */
	376	for (i = 0; i != 10; i++)
	377	test_ioctl_ioas_map(buffer, PAGE_SIZE, &iovas[i]);
	378	for (i = 0; i != 10; i++)
	379	test_ioctl_ioas_unmap(iovas[i], PAGE_SIZE);
	380
	381	/* Kernel automatically aligns IOVAs properly */
	382	for (i = 0; i != 10; i++) {
	383	size_t length = PAGE_SIZE * (i + 1);
	384
	385	if (self->domain_id) {
	386	test_ioctl_ioas_map(buffer, length, &iovas[i]);
	387	} else {
	388	test_ioctl_ioas_map((void *)(1UL << 31), length,
	389	&iovas[i]);
	390	}
	391	EXPECT_EQ(0, iovas[i] % (1UL << (ffs(length) - 1)));
	392	}
	393	for (i = 0; i != 10; i++)
	394	test_ioctl_ioas_unmap(iovas[i], PAGE_SIZE * (i + 1));
	395
	396	/* Avoids a reserved region */
	397	ASSERT_EQ(0,
	398	ioctl(self->fd, _IOMMU_TEST_CMD(IOMMU_TEST_OP_ADD_RESERVED),
	399	&test_cmd));
	400	for (i = 0; i != 10; i++) {
	401	size_t length = PAGE_SIZE * (i + 1);
	402
	403	test_ioctl_ioas_map(buffer, length, &iovas[i]);
	404	EXPECT_EQ(0, iovas[i] % (1UL << (ffs(length) - 1)));
	405	EXPECT_EQ(false,
	406	iovas[i] > test_cmd.add_reserved.start &&
	407	iovas[i] <
	408	test_cmd.add_reserved.start +
	409	test_cmd.add_reserved.length);
	410	}
	411	for (i = 0; i != 10; i++)
	412	test_ioctl_ioas_unmap(iovas[i], PAGE_SIZE * (i + 1));
	413
	414	/* Allowed region intersects with a reserved region */
	415	ranges[0].start = PAGE_SIZE;
	416	ranges[0].last = PAGE_SIZE * 600;
	417	EXPECT_ERRNO(EADDRINUSE,
	418	ioctl(self->fd, IOMMU_IOAS_ALLOW_IOVAS, &allow_cmd));
	419
	420	/* Allocate from an allowed region */
	421	if (self->domain_id) {
	422	ranges[0].start = MOCK_APERTURE_START + PAGE_SIZE;
	423	ranges[0].last = MOCK_APERTURE_START + PAGE_SIZE * 600 - 1;
	424	} else {
	425	ranges[0].start = PAGE_SIZE * 200;
	426	ranges[0].last = PAGE_SIZE * 600 - 1;
	427	}
	428	ASSERT_EQ(0, ioctl(self->fd, IOMMU_IOAS_ALLOW_IOVAS, &allow_cmd));
	429	for (i = 0; i != 10; i++) {
	430	size_t length = PAGE_SIZE * (i + 1);
	431
	432	test_ioctl_ioas_map(buffer, length, &iovas[i]);
	433	EXPECT_EQ(0, iovas[i] % (1UL << (ffs(length) - 1)));
	434	EXPECT_EQ(true, iovas[i] >= ranges[0].start);
	435	EXPECT_EQ(true, iovas[i] <= ranges[0].last);
	436	EXPECT_EQ(true, iovas[i] + length > ranges[0].start);
	437	EXPECT_EQ(true, iovas[i] + length <= ranges[0].last + 1);
	438	}
	439	for (i = 0; i != 10; i++)
	440	test_ioctl_ioas_unmap(iovas[i], PAGE_SIZE * (i + 1));
	441	}
	442
	443	TEST_F(iommufd_ioas, area_allowed)
	444	{
	445	struct iommu_test_cmd test_cmd = {
	446	.size = sizeof(test_cmd),
	447	.op = IOMMU_TEST_OP_ADD_RESERVED,
	448	.id = self->ioas_id,
	449	.add_reserved = { .start = PAGE_SIZE * 4,
	450	.length = PAGE_SIZE * 100 },
	451	};
	452	struct iommu_iova_range ranges[1] = {};
	453	struct iommu_ioas_allow_iovas allow_cmd = {
	454	.size = sizeof(allow_cmd),
	455	.ioas_id = self->ioas_id,
	456	.num_iovas = 1,
	457	.allowed_iovas = (uintptr_t)ranges,
	458	};
	459
	460	/* Reserved intersects an allowed */
	461	allow_cmd.num_iovas = 1;
	462	ranges[0].start = self->base_iova;
	463	ranges[0].last = ranges[0].start + PAGE_SIZE * 600;
	464	ASSERT_EQ(0, ioctl(self->fd, IOMMU_IOAS_ALLOW_IOVAS, &allow_cmd));
	465	test_cmd.add_reserved.start = ranges[0].start + PAGE_SIZE;
	466	test_cmd.add_reserved.length = PAGE_SIZE;
	467	EXPECT_ERRNO(EADDRINUSE,
	468	ioctl(self->fd,
	469	_IOMMU_TEST_CMD(IOMMU_TEST_OP_ADD_RESERVED),
	470	&test_cmd));
	471	allow_cmd.num_iovas = 0;
	472	ASSERT_EQ(0, ioctl(self->fd, IOMMU_IOAS_ALLOW_IOVAS, &allow_cmd));
	473
	474	/* Allowed intersects a reserved */
	475	ASSERT_EQ(0,
	476	ioctl(self->fd, _IOMMU_TEST_CMD(IOMMU_TEST_OP_ADD_RESERVED),
	477	&test_cmd));
	478	allow_cmd.num_iovas = 1;
	479	ranges[0].start = self->base_iova;
	480	ranges[0].last = ranges[0].start + PAGE_SIZE * 600;
	481	EXPECT_ERRNO(EADDRINUSE,
	482	ioctl(self->fd, IOMMU_IOAS_ALLOW_IOVAS, &allow_cmd));
	483	}
	484
	485	TEST_F(iommufd_ioas, copy_area)
	486	{
	487	struct iommu_ioas_copy copy_cmd = {
	488	.size = sizeof(copy_cmd),
	489	.flags = IOMMU_IOAS_MAP_FIXED_IOVA,
	490	.dst_ioas_id = self->ioas_id,
	491	.src_ioas_id = self->ioas_id,
	492	.length = PAGE_SIZE,
	493	};
	494
	495	test_ioctl_ioas_map_fixed(buffer, PAGE_SIZE, self->base_iova);
	496
	497	/* Copy inside a single IOAS */
	498	copy_cmd.src_iova = self->base_iova;
	499	copy_cmd.dst_iova = self->base_iova + PAGE_SIZE;
	500	ASSERT_EQ(0, ioctl(self->fd, IOMMU_IOAS_COPY, &copy_cmd));
	501
	502	/* Copy between IOAS's */
	503	copy_cmd.src_iova = self->base_iova;
	504	copy_cmd.dst_iova = 0;
	505	test_ioctl_ioas_alloc(&copy_cmd.dst_ioas_id);
	506	ASSERT_EQ(0, ioctl(self->fd, IOMMU_IOAS_COPY, &copy_cmd));
	507	}
	508
	509	TEST_F(iommufd_ioas, iova_ranges)
	510	{
	511	struct iommu_test_cmd test_cmd = {
	512	.size = sizeof(test_cmd),
	513	.op = IOMMU_TEST_OP_ADD_RESERVED,
	514	.id = self->ioas_id,
	515	.add_reserved = { .start = PAGE_SIZE, .length = PAGE_SIZE },
	516	};
	517	struct iommu_iova_range *ranges = buffer;
	518	struct iommu_ioas_iova_ranges ranges_cmd = {
	519	.size = sizeof(ranges_cmd),
	520	.ioas_id = self->ioas_id,
	521	.num_iovas = BUFFER_SIZE / sizeof(*ranges),
	522	.allowed_iovas = (uintptr_t)ranges,
	523	};
	524
	525	/* Range can be read */
	526	ASSERT_EQ(0, ioctl(self->fd, IOMMU_IOAS_IOVA_RANGES, &ranges_cmd));
	527	EXPECT_EQ(1, ranges_cmd.num_iovas);
	528	if (!self->domain_id) {
	529	EXPECT_EQ(0, ranges[0].start);
	530	EXPECT_EQ(SIZE_MAX, ranges[0].last);
	531	EXPECT_EQ(1, ranges_cmd.out_iova_alignment);
	532	} else {
	533	EXPECT_EQ(MOCK_APERTURE_START, ranges[0].start);
	534	EXPECT_EQ(MOCK_APERTURE_LAST, ranges[0].last);
	535	EXPECT_EQ(MOCK_PAGE_SIZE, ranges_cmd.out_iova_alignment);
	536	}
	537
	538	/* Buffer too small */
	539	memset(ranges, 0, BUFFER_SIZE);
	540	ranges_cmd.num_iovas = 0;
	541	EXPECT_ERRNO(EMSGSIZE,
	542	ioctl(self->fd, IOMMU_IOAS_IOVA_RANGES, &ranges_cmd));
	543	EXPECT_EQ(1, ranges_cmd.num_iovas);
	544	EXPECT_EQ(0, ranges[0].start);
	545	EXPECT_EQ(0, ranges[0].last);
	546
	547	/* 2 ranges */
	548	ASSERT_EQ(0,
	549	ioctl(self->fd, _IOMMU_TEST_CMD(IOMMU_TEST_OP_ADD_RESERVED),
	550	&test_cmd));
	551	ranges_cmd.num_iovas = BUFFER_SIZE / sizeof(*ranges);
	552	ASSERT_EQ(0, ioctl(self->fd, IOMMU_IOAS_IOVA_RANGES, &ranges_cmd));
	553	if (!self->domain_id) {
	554	EXPECT_EQ(2, ranges_cmd.num_iovas);
	555	EXPECT_EQ(0, ranges[0].start);
	556	EXPECT_EQ(PAGE_SIZE - 1, ranges[0].last);
	557	EXPECT_EQ(PAGE_SIZE * 2, ranges[1].start);
	558	EXPECT_EQ(SIZE_MAX, ranges[1].last);
	559	} else {
	560	EXPECT_EQ(1, ranges_cmd.num_iovas);
	561	EXPECT_EQ(MOCK_APERTURE_START, ranges[0].start);
	562	EXPECT_EQ(MOCK_APERTURE_LAST, ranges[0].last);
	563	}
	564
	565	/* Buffer too small */
	566	memset(ranges, 0, BUFFER_SIZE);
	567	ranges_cmd.num_iovas = 1;
	568	if (!self->domain_id) {
	569	EXPECT_ERRNO(EMSGSIZE, ioctl(self->fd, IOMMU_IOAS_IOVA_RANGES,
	570	&ranges_cmd));
	571	EXPECT_EQ(2, ranges_cmd.num_iovas);
	572	EXPECT_EQ(0, ranges[0].start);
	573	EXPECT_EQ(PAGE_SIZE - 1, ranges[0].last);
	574	} else {
	575	ASSERT_EQ(0,
	576	ioctl(self->fd, IOMMU_IOAS_IOVA_RANGES, &ranges_cmd));
	577	EXPECT_EQ(1, ranges_cmd.num_iovas);
	578	EXPECT_EQ(MOCK_APERTURE_START, ranges[0].start);
	579	EXPECT_EQ(MOCK_APERTURE_LAST, ranges[0].last);
	580	}
	581	EXPECT_EQ(0, ranges[1].start);
	582	EXPECT_EQ(0, ranges[1].last);
	583	}
	584
	585	TEST_F(iommufd_ioas, access_pin)
	586	{
	587	struct iommu_test_cmd access_cmd = {
	588	.size = sizeof(access_cmd),
	589	.op = IOMMU_TEST_OP_ACCESS_PAGES,
	590	.access_pages = { .iova = MOCK_APERTURE_START,
	591	.length = BUFFER_SIZE,
	592	.uptr = (uintptr_t)buffer },
	593	};
	594	struct iommu_test_cmd check_map_cmd = {
	595	.size = sizeof(check_map_cmd),
	596	.op = IOMMU_TEST_OP_MD_CHECK_MAP,
	597	.check_map = { .iova = MOCK_APERTURE_START,
	598	.length = BUFFER_SIZE,
	599	.uptr = (uintptr_t)buffer },
	600	};
	601	uint32_t access_pages_id;
	602	unsigned int npages;
	603
	604	test_cmd_create_access(self->ioas_id, &access_cmd.id,
	605	MOCK_FLAGS_ACCESS_CREATE_NEEDS_PIN_PAGES);
	606
	607	for (npages = 1; npages < BUFFER_SIZE / PAGE_SIZE; npages++) {
	608	uint32_t mock_device_id;
	609	uint32_t mock_hwpt_id;
	610
	611	access_cmd.access_pages.length = npages * PAGE_SIZE;
	612
	613	/* Single map/unmap */
	614	test_ioctl_ioas_map_fixed(buffer, BUFFER_SIZE,
	615	MOCK_APERTURE_START);
	616	ASSERT_EQ(0, ioctl(self->fd,
	617	_IOMMU_TEST_CMD(IOMMU_TEST_OP_ACCESS_PAGES),
	618	&access_cmd));
	619	test_cmd_destroy_access_pages(
	620	access_cmd.id,
	621	access_cmd.access_pages.out_access_pages_id);
	622
	623	/* Double user */
	624	ASSERT_EQ(0, ioctl(self->fd,
	625	_IOMMU_TEST_CMD(IOMMU_TEST_OP_ACCESS_PAGES),
	626	&access_cmd));
	627	access_pages_id = access_cmd.access_pages.out_access_pages_id;
	628	ASSERT_EQ(0, ioctl(self->fd,
	629	_IOMMU_TEST_CMD(IOMMU_TEST_OP_ACCESS_PAGES),
	630	&access_cmd));
	631	test_cmd_destroy_access_pages(
	632	access_cmd.id,
	633	access_cmd.access_pages.out_access_pages_id);
	634	test_cmd_destroy_access_pages(access_cmd.id, access_pages_id);
	635
	636	/* Add/remove a domain with a user */
	637	ASSERT_EQ(0, ioctl(self->fd,
	638	_IOMMU_TEST_CMD(IOMMU_TEST_OP_ACCESS_PAGES),
	639	&access_cmd));
	640	test_cmd_mock_domain(self->ioas_id, &mock_device_id,
	641	&mock_hwpt_id);
	642	check_map_cmd.id = mock_hwpt_id;
	643	ASSERT_EQ(0, ioctl(self->fd,
	644	_IOMMU_TEST_CMD(IOMMU_TEST_OP_MD_CHECK_MAP),
	645	&check_map_cmd));
	646
	647	test_ioctl_destroy(mock_device_id);
	648	test_ioctl_destroy(mock_hwpt_id);
	649	test_cmd_destroy_access_pages(
	650	access_cmd.id,
	651	access_cmd.access_pages.out_access_pages_id);
	652
	653	test_ioctl_ioas_unmap(MOCK_APERTURE_START, BUFFER_SIZE);
	654	}
	655	test_cmd_destroy_access(access_cmd.id);
	656	}
	657
	658	TEST_F(iommufd_ioas, access_pin_unmap)
	659	{
	660	struct iommu_test_cmd access_pages_cmd = {
	661	.size = sizeof(access_pages_cmd),
	662	.op = IOMMU_TEST_OP_ACCESS_PAGES,
	663	.access_pages = { .iova = MOCK_APERTURE_START,
	664	.length = BUFFER_SIZE,
	665	.uptr = (uintptr_t)buffer },
	666	};
	667
	668	test_cmd_create_access(self->ioas_id, &access_pages_cmd.id,
	669	MOCK_FLAGS_ACCESS_CREATE_NEEDS_PIN_PAGES);
	670	test_ioctl_ioas_map_fixed(buffer, BUFFER_SIZE, MOCK_APERTURE_START);
	671	ASSERT_EQ(0,
	672	ioctl(self->fd, _IOMMU_TEST_CMD(IOMMU_TEST_OP_ACCESS_PAGES),
	673	&access_pages_cmd));
	674
	675	/* Trigger the unmap op */
	676	test_ioctl_ioas_unmap(MOCK_APERTURE_START, BUFFER_SIZE);
	677
	678	/* kernel removed the item for us */
	679	test_err_destroy_access_pages(
	680	ENOENT, access_pages_cmd.id,
	681	access_pages_cmd.access_pages.out_access_pages_id);
	682	}
	683
	684	static void check_access_rw(struct __test_metadata *_metadata, int fd,
	685	unsigned int access_id, uint64_t iova,
	686	unsigned int def_flags)
	687	{
	688	uint16_t tmp[32];
	689	struct iommu_test_cmd access_cmd = {
	690	.size = sizeof(access_cmd),
	691	.op = IOMMU_TEST_OP_ACCESS_RW,
	692	.id = access_id,
	693	.access_rw = { .uptr = (uintptr_t)tmp },
	694	};
	695	uint16_t *buffer16 = buffer;
	696	unsigned int i;
	697	void *tmp2;
	698
	699	for (i = 0; i != BUFFER_SIZE / sizeof(*buffer16); i++)
	700	buffer16[i] = rand();
	701
	702	for (access_cmd.access_rw.iova = iova + PAGE_SIZE - 50;
	703	access_cmd.access_rw.iova < iova + PAGE_SIZE + 50;
	704	access_cmd.access_rw.iova++) {
	705	for (access_cmd.access_rw.length = 1;
	706	access_cmd.access_rw.length < sizeof(tmp);
	707	access_cmd.access_rw.length++) {
	708	access_cmd.access_rw.flags = def_flags;
	709	ASSERT_EQ(0, ioctl(fd,
	710	_IOMMU_TEST_CMD(
	711	IOMMU_TEST_OP_ACCESS_RW),
	712	&access_cmd));
	713	ASSERT_EQ(0,
	714	memcmp(buffer + (access_cmd.access_rw.iova -
	715	iova),
	716	tmp, access_cmd.access_rw.length));
	717
	718	for (i = 0; i != ARRAY_SIZE(tmp); i++)
	719	tmp[i] = rand();
	720	access_cmd.access_rw.flags = def_flags \|
	721	MOCK_ACCESS_RW_WRITE;
	722	ASSERT_EQ(0, ioctl(fd,
	723	_IOMMU_TEST_CMD(
	724	IOMMU_TEST_OP_ACCESS_RW),
	725	&access_cmd));
	726	ASSERT_EQ(0,
	727	memcmp(buffer + (access_cmd.access_rw.iova -
	728	iova),
	729	tmp, access_cmd.access_rw.length));
	730	}
	731	}
	732
	733	/* Multi-page test */
	734	tmp2 = malloc(BUFFER_SIZE);
	735	ASSERT_NE(NULL, tmp2);
	736	access_cmd.access_rw.iova = iova;
	737	access_cmd.access_rw.length = BUFFER_SIZE;
	738	access_cmd.access_rw.flags = def_flags;
	739	access_cmd.access_rw.uptr = (uintptr_t)tmp2;
	740	ASSERT_EQ(0, ioctl(fd, _IOMMU_TEST_CMD(IOMMU_TEST_OP_ACCESS_RW),
	741	&access_cmd));
	742	ASSERT_EQ(0, memcmp(buffer, tmp2, access_cmd.access_rw.length));
	743	free(tmp2);
	744	}
	745
	746	TEST_F(iommufd_ioas, access_rw)
	747	{
	748	__u32 access_id;
	749	__u64 iova;
	750
	751	test_cmd_create_access(self->ioas_id, &access_id, 0);
	752	test_ioctl_ioas_map(buffer, BUFFER_SIZE, &iova);
	753	check_access_rw(_metadata, self->fd, access_id, iova, 0);
	754	check_access_rw(_metadata, self->fd, access_id, iova,
	755	MOCK_ACCESS_RW_SLOW_PATH);
	756	test_ioctl_ioas_unmap(iova, BUFFER_SIZE);
	757	test_cmd_destroy_access(access_id);
	758	}
	759
	760	TEST_F(iommufd_ioas, access_rw_unaligned)
	761	{
	762	__u32 access_id;
	763	__u64 iova;
	764
	765	test_cmd_create_access(self->ioas_id, &access_id, 0);
	766
	767	/* Unaligned pages */
	768	iova = self->base_iova + MOCK_PAGE_SIZE;
	769	test_ioctl_ioas_map_fixed(buffer, BUFFER_SIZE, iova);
	770	check_access_rw(_metadata, self->fd, access_id, iova, 0);
	771	test_ioctl_ioas_unmap(iova, BUFFER_SIZE);
	772	test_cmd_destroy_access(access_id);
	773	}
	774
	775	TEST_F(iommufd_ioas, fork_gone)
	776	{
	777	__u32 access_id;
	778	pid_t child;
	779
	780	test_cmd_create_access(self->ioas_id, &access_id, 0);
	781
	782	/* Create a mapping with a different mm */
	783	child = fork();
	784	if (!child) {
	785	test_ioctl_ioas_map_fixed(buffer, BUFFER_SIZE,
	786	MOCK_APERTURE_START);
	787	exit(0);
	788	}
	789	ASSERT_NE(-1, child);
	790	ASSERT_EQ(child, waitpid(child, NULL, 0));
	791
	792	if (self->domain_id) {
	793	/*
	794	* If a domain already existed then everything was pinned within
	795	* the fork, so this copies from one domain to another.
	796	*/
	797	test_cmd_mock_domain(self->ioas_id, NULL, NULL);
	798	check_access_rw(_metadata, self->fd, access_id,
	799	MOCK_APERTURE_START, 0);
	800
	801	} else {
	802	/*
	803	* Otherwise we need to actually pin pages which can't happen
	804	* since the fork is gone.
	805	*/
	806	test_err_mock_domain(EFAULT, self->ioas_id, NULL, NULL);
	807	}
	808
	809	test_cmd_destroy_access(access_id);
	810	}
	811
	812	TEST_F(iommufd_ioas, fork_present)
	813	{
	814	__u32 access_id;
	815	int pipefds[2];
	816	uint64_t tmp;
	817	pid_t child;
	818	int efd;
	819
	820	test_cmd_create_access(self->ioas_id, &access_id, 0);
	821
	822	ASSERT_EQ(0, pipe2(pipefds, O_CLOEXEC));
	823	efd = eventfd(0, EFD_CLOEXEC);
	824	ASSERT_NE(-1, efd);
	825
	826	/* Create a mapping with a different mm */
	827	child = fork();
	828	if (!child) {
	829	__u64 iova;
	830	uint64_t one = 1;
	831
	832	close(pipefds[1]);
	833	test_ioctl_ioas_map_fixed(buffer, BUFFER_SIZE,
	834	MOCK_APERTURE_START);
	835	if (write(efd, &one, sizeof(one)) != sizeof(one))
	836	exit(100);
	837	if (read(pipefds[0], &iova, 1) != 1)
	838	exit(100);
	839	exit(0);
	840	}
	841	close(pipefds[0]);
	842	ASSERT_NE(-1, child);
	843	ASSERT_EQ(8, read(efd, &tmp, sizeof(tmp)));
	844
	845	/* Read pages from the remote process */
	846	test_cmd_mock_domain(self->ioas_id, NULL, NULL);
	847	check_access_rw(_metadata, self->fd, access_id, MOCK_APERTURE_START, 0);
	848
	849	ASSERT_EQ(0, close(pipefds[1]));
	850	ASSERT_EQ(child, waitpid(child, NULL, 0));
	851
	852	test_cmd_destroy_access(access_id);
	853	}
	854
	855	TEST_F(iommufd_ioas, ioas_option_huge_pages)
	856	{
	857	struct iommu_option cmd = {
	858	.size = sizeof(cmd),
	859	.option_id = IOMMU_OPTION_HUGE_PAGES,
	860	.op = IOMMU_OPTION_OP_GET,
	861	.val64 = 3,
	862	.object_id = self->ioas_id,
	863	};
	864
	865	ASSERT_EQ(0, ioctl(self->fd, IOMMU_OPTION, &cmd));
	866	ASSERT_EQ(1, cmd.val64);
	867
	868	cmd.op = IOMMU_OPTION_OP_SET;
	869	cmd.val64 = 0;
	870	ASSERT_EQ(0, ioctl(self->fd, IOMMU_OPTION, &cmd));
	871
	872	cmd.op = IOMMU_OPTION_OP_GET;
	873	cmd.val64 = 3;
	874	ASSERT_EQ(0, ioctl(self->fd, IOMMU_OPTION, &cmd));
	875	ASSERT_EQ(0, cmd.val64);
	876
	877	cmd.op = IOMMU_OPTION_OP_SET;
	878	cmd.val64 = 2;
	879	EXPECT_ERRNO(EINVAL, ioctl(self->fd, IOMMU_OPTION, &cmd));
	880
	881	cmd.op = IOMMU_OPTION_OP_SET;
	882	cmd.val64 = 1;
	883	ASSERT_EQ(0, ioctl(self->fd, IOMMU_OPTION, &cmd));
	884	}
	885
	886	TEST_F(iommufd_ioas, ioas_iova_alloc)
	887	{
	888	unsigned int length;
	889	__u64 iova;
	890
	891	for (length = 1; length != PAGE_SIZE * 2; length++) {
	892	if (variant->mock_domains && (length % MOCK_PAGE_SIZE)) {
	893	test_err_ioctl_ioas_map(EINVAL, buffer, length, &iova);
	894	} else {
	895	test_ioctl_ioas_map(buffer, length, &iova);
	896	test_ioctl_ioas_unmap(iova, length);
	897	}
	898	}
	899	}
	900
	901	TEST_F(iommufd_ioas, ioas_align_change)
	902	{
	903	struct iommu_option cmd = {
	904	.size = sizeof(cmd),
	905	.option_id = IOMMU_OPTION_HUGE_PAGES,
	906	.op = IOMMU_OPTION_OP_SET,
	907	.object_id = self->ioas_id,
	908	/* 0 means everything must be aligned to PAGE_SIZE */
	909	.val64 = 0,
	910	};
	911
	912	/*
	913	* We cannot upgrade the alignment using OPTION_HUGE_PAGES when a domain
	914	* and map are present.
	915	*/
	916	if (variant->mock_domains)
	917	return;
	918
	919	/*
	920	* We can upgrade to PAGE_SIZE alignment when things are aligned right
	921	*/
	922	test_ioctl_ioas_map_fixed(buffer, PAGE_SIZE, MOCK_APERTURE_START);
	923	ASSERT_EQ(0, ioctl(self->fd, IOMMU_OPTION, &cmd));
	924
	925	/* Misalignment is rejected at map time */
	926	test_err_ioctl_ioas_map_fixed(EINVAL, buffer + MOCK_PAGE_SIZE,
	927	PAGE_SIZE,
	928	MOCK_APERTURE_START + PAGE_SIZE);
	929	ASSERT_EQ(0, ioctl(self->fd, IOMMU_OPTION, &cmd));
	930
	931	/* Reduce alignment */
	932	cmd.val64 = 1;
	933	ASSERT_EQ(0, ioctl(self->fd, IOMMU_OPTION, &cmd));
	934
	935	/* Confirm misalignment is rejected during alignment upgrade */
	936	test_ioctl_ioas_map_fixed(buffer + MOCK_PAGE_SIZE, PAGE_SIZE,
	937	MOCK_APERTURE_START + PAGE_SIZE);
	938	cmd.val64 = 0;
	939	EXPECT_ERRNO(EADDRINUSE, ioctl(self->fd, IOMMU_OPTION, &cmd));
	940
	941	test_ioctl_ioas_unmap(MOCK_APERTURE_START + PAGE_SIZE, PAGE_SIZE);
	942	test_ioctl_ioas_unmap(MOCK_APERTURE_START, PAGE_SIZE);
	943	}
	944
	945	TEST_F(iommufd_ioas, copy_sweep)
	946	{
	947	struct iommu_ioas_copy copy_cmd = {
	948	.size = sizeof(copy_cmd),
	949	.flags = IOMMU_IOAS_MAP_FIXED_IOVA,
	950	.src_ioas_id = self->ioas_id,
	951	.dst_iova = MOCK_APERTURE_START,
	952	.length = MOCK_PAGE_SIZE,
	953	};
	954	unsigned int dst_ioas_id;
	955	uint64_t last_iova;
	956	uint64_t iova;
	957
	958	test_ioctl_ioas_alloc(&dst_ioas_id);
	959	copy_cmd.dst_ioas_id = dst_ioas_id;
	960
	961	if (variant->mock_domains)
	962	last_iova = MOCK_APERTURE_START + BUFFER_SIZE - 1;
	963	else
	964	last_iova = MOCK_APERTURE_START + BUFFER_SIZE - 2;
	965
	966	test_ioctl_ioas_map_fixed(buffer, last_iova - MOCK_APERTURE_START + 1,
	967	MOCK_APERTURE_START);
	968
	969	for (iova = MOCK_APERTURE_START - PAGE_SIZE; iova <= last_iova;
	970	iova += 511) {
	971	copy_cmd.src_iova = iova;
	972	if (iova < MOCK_APERTURE_START \|\|
	973	iova + copy_cmd.length - 1 > last_iova) {
	974	EXPECT_ERRNO(ENOENT, ioctl(self->fd, IOMMU_IOAS_COPY,
	975	&copy_cmd));
	976	} else {
	977	ASSERT_EQ(0,
	978	ioctl(self->fd, IOMMU_IOAS_COPY, &copy_cmd));
	979	test_ioctl_ioas_unmap_id(dst_ioas_id, copy_cmd.dst_iova,
	980	copy_cmd.length);
	981	}
	982	}
	983
	984	test_ioctl_destroy(dst_ioas_id);
	985	}
	986
	987	FIXTURE(iommufd_mock_domain)
	988	{
	989	int fd;
	990	uint32_t ioas_id;
	991	uint32_t domain_id;
	992	uint32_t domain_ids[2];
	993	int mmap_flags;
	994	size_t mmap_buf_size;
	995	};
	996
	997	FIXTURE_VARIANT(iommufd_mock_domain)
	998	{
	999	unsigned int mock_domains;
	1000	bool hugepages;
	1001	};
	1002
	1003	FIXTURE_SETUP(iommufd_mock_domain)
	1004	{
	1005	unsigned int i;
	1006
	1007	self->fd = open("/dev/iommu", O_RDWR);
	1008	ASSERT_NE(-1, self->fd);
	1009	test_ioctl_ioas_alloc(&self->ioas_id);
	1010
	1011	ASSERT_GE(ARRAY_SIZE(self->domain_ids), variant->mock_domains);
	1012
	1013	for (i = 0; i != variant->mock_domains; i++)
	1014	test_cmd_mock_domain(self->ioas_id, NULL, &self->domain_ids[i]);
	1015	self->domain_id = self->domain_ids[0];
	1016
	1017	self->mmap_flags = MAP_SHARED \| MAP_ANONYMOUS;
	1018	self->mmap_buf_size = PAGE_SIZE * 8;
	1019	if (variant->hugepages) {
	1020	/*
	1021	* MAP_POPULATE will cause the kernel to fail mmap if THPs are
	1022	* not available.
	1023	*/
	1024	self->mmap_flags \|= MAP_HUGETLB \| MAP_POPULATE;
	1025	self->mmap_buf_size = HUGEPAGE_SIZE * 2;
	1026	}
	1027	}
	1028
	1029	FIXTURE_TEARDOWN(iommufd_mock_domain)
	1030	{
	1031	teardown_iommufd(self->fd, _metadata);
	1032	}
	1033
	1034	FIXTURE_VARIANT_ADD(iommufd_mock_domain, one_domain)
	1035	{
	1036	.mock_domains = 1,
	1037	.hugepages = false,
	1038	};
	1039
	1040	FIXTURE_VARIANT_ADD(iommufd_mock_domain, two_domains)
	1041	{
	1042	.mock_domains = 2,
	1043	.hugepages = false,
	1044	};
	1045
	1046	FIXTURE_VARIANT_ADD(iommufd_mock_domain, one_domain_hugepage)
	1047	{
	1048	.mock_domains = 1,
	1049	.hugepages = true,
	1050	};
	1051
	1052	FIXTURE_VARIANT_ADD(iommufd_mock_domain, two_domains_hugepage)
	1053	{
	1054	.mock_domains = 2,
	1055	.hugepages = true,
	1056	};
	1057
	1058	/* Have the kernel check that the user pages made it to the iommu_domain */
	1059	#define check_mock_iova(_ptr, _iova, _length) \
	1060	({ \
	1061	struct iommu_test_cmd check_map_cmd = { \
	1062	.size = sizeof(check_map_cmd), \
	1063	.op = IOMMU_TEST_OP_MD_CHECK_MAP, \
	1064	.id = self->domain_id, \
	1065	.check_map = { .iova = _iova, \
	1066	.length = _length, \
	1067	.uptr = (uintptr_t)(_ptr) }, \
	1068	}; \
	1069	ASSERT_EQ(0, \
	1070	ioctl(self->fd, \
	1071	_IOMMU_TEST_CMD(IOMMU_TEST_OP_MD_CHECK_MAP), \
	1072	&check_map_cmd)); \
	1073	if (self->domain_ids[1]) { \
	1074	check_map_cmd.id = self->domain_ids[1]; \
	1075	ASSERT_EQ(0, \
	1076	ioctl(self->fd, \
	1077	_IOMMU_TEST_CMD( \
	1078	IOMMU_TEST_OP_MD_CHECK_MAP), \
	1079	&check_map_cmd)); \
	1080	} \
	1081	})
	1082
	1083	TEST_F(iommufd_mock_domain, basic)
	1084	{
	1085	size_t buf_size = self->mmap_buf_size;
	1086	uint8_t *buf;
	1087	__u64 iova;
	1088
	1089	/* Simple one page map */
	1090	test_ioctl_ioas_map(buffer, PAGE_SIZE, &iova);
	1091	check_mock_iova(buffer, iova, PAGE_SIZE);
	1092
	1093	buf = mmap(0, buf_size, PROT_READ \| PROT_WRITE, self->mmap_flags, -1,
	1094	0);
	1095	ASSERT_NE(MAP_FAILED, buf);
	1096
	1097	/* EFAULT half way through mapping */
	1098	ASSERT_EQ(0, munmap(buf + buf_size / 2, buf_size / 2));
	1099	test_err_ioctl_ioas_map(EFAULT, buf, buf_size, &iova);
	1100
	1101	/* EFAULT on first page */
	1102	ASSERT_EQ(0, munmap(buf, buf_size / 2));
	1103	test_err_ioctl_ioas_map(EFAULT, buf, buf_size, &iova);
	1104	}
	1105
	1106	TEST_F(iommufd_mock_domain, ro_unshare)
	1107	{
	1108	uint8_t *buf;
	1109	__u64 iova;
	1110	int fd;
	1111
	1112	fd = open("/proc/self/exe", O_RDONLY);
	1113	ASSERT_NE(-1, fd);
	1114
	1115	buf = mmap(0, PAGE_SIZE, PROT_READ \| PROT_WRITE, MAP_PRIVATE, fd, 0);
	1116	ASSERT_NE(MAP_FAILED, buf);
	1117	close(fd);
	1118
	1119	/*
	1120	* There have been lots of changes to the "unshare" mechanism in
	1121	* get_user_pages(), make sure it works right. The write to the page
	1122	* after we map it for reading should not change the assigned PFN.
	1123	*/
	1124	ASSERT_EQ(0,
	1125	_test_ioctl_ioas_map(self->fd, self->ioas_id, buf, PAGE_SIZE,
	1126	&iova, IOMMU_IOAS_MAP_READABLE));
	1127	check_mock_iova(buf, iova, PAGE_SIZE);
	1128	memset(buf, 1, PAGE_SIZE);
	1129	check_mock_iova(buf, iova, PAGE_SIZE);
	1130	ASSERT_EQ(0, munmap(buf, PAGE_SIZE));
	1131	}
	1132
	1133	TEST_F(iommufd_mock_domain, all_aligns)
	1134	{
	1135	size_t test_step = variant->hugepages ? (self->mmap_buf_size / 16) :
	1136	MOCK_PAGE_SIZE;
	1137	size_t buf_size = self->mmap_buf_size;
	1138	unsigned int start;
	1139	unsigned int end;
	1140	uint8_t *buf;
	1141
	1142	buf = mmap(0, buf_size, PROT_READ \| PROT_WRITE, self->mmap_flags, -1,
	1143	0);
	1144	ASSERT_NE(MAP_FAILED, buf);
	1145	check_refs(buf, buf_size, 0);
	1146
	1147	/*
	1148	* Map every combination of page size and alignment within a big region,
	1149	* less for hugepage case as it takes so long to finish.
	1150	*/
	1151	for (start = 0; start < buf_size; start += test_step) {
	1152	if (variant->hugepages)
	1153	end = buf_size;
	1154	else
	1155	end = start + MOCK_PAGE_SIZE;
	1156	for (; end < buf_size; end += MOCK_PAGE_SIZE) {
	1157	size_t length = end - start;
	1158	__u64 iova;
	1159
	1160	test_ioctl_ioas_map(buf + start, length, &iova);
	1161	check_mock_iova(buf + start, iova, length);
	1162	check_refs(buf + start / PAGE_SIZE * PAGE_SIZE,
	1163	end / PAGE_SIZE * PAGE_SIZE -
	1164	start / PAGE_SIZE * PAGE_SIZE,
	1165	1);
	1166
	1167	test_ioctl_ioas_unmap(iova, length);
	1168	}
	1169	}
	1170	check_refs(buf, buf_size, 0);
	1171	ASSERT_EQ(0, munmap(buf, buf_size));
	1172	}
	1173
	1174	TEST_F(iommufd_mock_domain, all_aligns_copy)
	1175	{
	1176	size_t test_step = variant->hugepages ? self->mmap_buf_size / 16 :
	1177	MOCK_PAGE_SIZE;
	1178	size_t buf_size = self->mmap_buf_size;
	1179	unsigned int start;
	1180	unsigned int end;
	1181	uint8_t *buf;
	1182
	1183	buf = mmap(0, buf_size, PROT_READ \| PROT_WRITE, self->mmap_flags, -1,
	1184	0);
	1185	ASSERT_NE(MAP_FAILED, buf);
	1186	check_refs(buf, buf_size, 0);
	1187
	1188	/*
	1189	* Map every combination of page size and alignment within a big region,
	1190	* less for hugepage case as it takes so long to finish.
	1191	*/
	1192	for (start = 0; start < buf_size; start += test_step) {
	1193	if (variant->hugepages)
	1194	end = buf_size;
	1195	else
	1196	end = start + MOCK_PAGE_SIZE;
	1197	for (; end < buf_size; end += MOCK_PAGE_SIZE) {
	1198	size_t length = end - start;
	1199	unsigned int old_id;
	1200	uint32_t mock_device_id;
	1201	__u64 iova;
	1202
	1203	test_ioctl_ioas_map(buf + start, length, &iova);
	1204
	1205	/* Add and destroy a domain while the area exists */
	1206	old_id = self->domain_ids[1];
	1207	test_cmd_mock_domain(self->ioas_id, &mock_device_id,
	1208	&self->domain_ids[1]);
	1209
	1210	check_mock_iova(buf + start, iova, length);
	1211	check_refs(buf + start / PAGE_SIZE * PAGE_SIZE,
	1212	end / PAGE_SIZE * PAGE_SIZE -
	1213	start / PAGE_SIZE * PAGE_SIZE,
	1214	1);
	1215
	1216	test_ioctl_destroy(mock_device_id);
	1217	test_ioctl_destroy(self->domain_ids[1]);
	1218	self->domain_ids[1] = old_id;
	1219
	1220	test_ioctl_ioas_unmap(iova, length);
	1221	}
	1222	}
	1223	check_refs(buf, buf_size, 0);
	1224	ASSERT_EQ(0, munmap(buf, buf_size));
	1225	}
	1226
	1227	TEST_F(iommufd_mock_domain, user_copy)
	1228	{
	1229	struct iommu_test_cmd access_cmd = {
	1230	.size = sizeof(access_cmd),
	1231	.op = IOMMU_TEST_OP_ACCESS_PAGES,
	1232	.access_pages = { .length = BUFFER_SIZE,
	1233	.uptr = (uintptr_t)buffer },
	1234	};
	1235	struct iommu_ioas_copy copy_cmd = {
	1236	.size = sizeof(copy_cmd),
	1237	.flags = IOMMU_IOAS_MAP_FIXED_IOVA,
	1238	.dst_ioas_id = self->ioas_id,
	1239	.dst_iova = MOCK_APERTURE_START,
	1240	.length = BUFFER_SIZE,
	1241	};
	1242	unsigned int ioas_id;
	1243
	1244	/* Pin the pages in an IOAS with no domains then copy to an IOAS with domains */
	1245	test_ioctl_ioas_alloc(&ioas_id);
	1246	test_ioctl_ioas_map_id(ioas_id, buffer, BUFFER_SIZE,
	1247	&copy_cmd.src_iova);
	1248
	1249	test_cmd_create_access(ioas_id, &access_cmd.id,
	1250	MOCK_FLAGS_ACCESS_CREATE_NEEDS_PIN_PAGES);
	1251
	1252	access_cmd.access_pages.iova = copy_cmd.src_iova;
	1253	ASSERT_EQ(0,
	1254	ioctl(self->fd, _IOMMU_TEST_CMD(IOMMU_TEST_OP_ACCESS_PAGES),
	1255	&access_cmd));
	1256	copy_cmd.src_ioas_id = ioas_id;
	1257	ASSERT_EQ(0, ioctl(self->fd, IOMMU_IOAS_COPY, &copy_cmd));
	1258	check_mock_iova(buffer, MOCK_APERTURE_START, BUFFER_SIZE);
	1259
	1260	test_cmd_destroy_access_pages(
	1261	access_cmd.id, access_cmd.access_pages.out_access_pages_id);
	1262	test_cmd_destroy_access(access_cmd.id) test_ioctl_destroy(ioas_id);
	1263
	1264	test_ioctl_destroy(ioas_id);
	1265	}
	1266
	1267	/* VFIO compatibility IOCTLs */
	1268
	1269	TEST_F(iommufd, simple_ioctls)
	1270	{
	1271	ASSERT_EQ(VFIO_API_VERSION, ioctl(self->fd, VFIO_GET_API_VERSION));
	1272	ASSERT_EQ(1, ioctl(self->fd, VFIO_CHECK_EXTENSION, VFIO_TYPE1v2_IOMMU));
	1273	}
	1274
	1275	TEST_F(iommufd, unmap_cmd)
	1276	{
	1277	struct vfio_iommu_type1_dma_unmap unmap_cmd = {
	1278	.iova = MOCK_APERTURE_START,
	1279	.size = PAGE_SIZE,
	1280	};
	1281
	1282	unmap_cmd.argsz = 1;
	1283	EXPECT_ERRNO(EINVAL, ioctl(self->fd, VFIO_IOMMU_UNMAP_DMA, &unmap_cmd));
	1284
	1285	unmap_cmd.argsz = sizeof(unmap_cmd);
	1286	unmap_cmd.flags = 1 << 31;
	1287	EXPECT_ERRNO(EINVAL, ioctl(self->fd, VFIO_IOMMU_UNMAP_DMA, &unmap_cmd));
	1288
	1289	unmap_cmd.flags = 0;
	1290	EXPECT_ERRNO(ENODEV, ioctl(self->fd, VFIO_IOMMU_UNMAP_DMA, &unmap_cmd));
	1291	}
	1292
	1293	TEST_F(iommufd, map_cmd)
	1294	{
	1295	struct vfio_iommu_type1_dma_map map_cmd = {
	1296	.iova = MOCK_APERTURE_START,
	1297	.size = PAGE_SIZE,
	1298	.vaddr = (__u64)buffer,
	1299	};
	1300
	1301	map_cmd.argsz = 1;
	1302	EXPECT_ERRNO(EINVAL, ioctl(self->fd, VFIO_IOMMU_MAP_DMA, &map_cmd));
	1303
	1304	map_cmd.argsz = sizeof(map_cmd);
	1305	map_cmd.flags = 1 << 31;
	1306	EXPECT_ERRNO(EINVAL, ioctl(self->fd, VFIO_IOMMU_MAP_DMA, &map_cmd));
	1307
	1308	/* Requires a domain to be attached */
	1309	map_cmd.flags = VFIO_DMA_MAP_FLAG_READ \| VFIO_DMA_MAP_FLAG_WRITE;
	1310	EXPECT_ERRNO(ENODEV, ioctl(self->fd, VFIO_IOMMU_MAP_DMA, &map_cmd));
	1311	}
	1312
	1313	TEST_F(iommufd, info_cmd)
	1314	{
	1315	struct vfio_iommu_type1_info info_cmd = {};
	1316
	1317	/* Invalid argsz */
	1318	info_cmd.argsz = 1;
	1319	EXPECT_ERRNO(EINVAL, ioctl(self->fd, VFIO_IOMMU_GET_INFO, &info_cmd));
	1320
	1321	info_cmd.argsz = sizeof(info_cmd);
	1322	EXPECT_ERRNO(ENODEV, ioctl(self->fd, VFIO_IOMMU_GET_INFO, &info_cmd));
	1323	}
	1324
	1325	TEST_F(iommufd, set_iommu_cmd)
	1326	{
	1327	/* Requires a domain to be attached */
	1328	EXPECT_ERRNO(ENODEV,
	1329	ioctl(self->fd, VFIO_SET_IOMMU, VFIO_TYPE1v2_IOMMU));
	1330	EXPECT_ERRNO(ENODEV, ioctl(self->fd, VFIO_SET_IOMMU, VFIO_TYPE1_IOMMU));
	1331	}
	1332
	1333	TEST_F(iommufd, vfio_ioas)
	1334	{
	1335	struct iommu_vfio_ioas vfio_ioas_cmd = {
	1336	.size = sizeof(vfio_ioas_cmd),
	1337	.op = IOMMU_VFIO_IOAS_GET,
	1338	};
	1339	__u32 ioas_id;
	1340
	1341	/* ENODEV if there is no compat ioas */
	1342	EXPECT_ERRNO(ENODEV, ioctl(self->fd, IOMMU_VFIO_IOAS, &vfio_ioas_cmd));
	1343
	1344	/* Invalid id for set */
	1345	vfio_ioas_cmd.op = IOMMU_VFIO_IOAS_SET;
	1346	EXPECT_ERRNO(ENOENT, ioctl(self->fd, IOMMU_VFIO_IOAS, &vfio_ioas_cmd));
	1347
	1348	/* Valid id for set*/
	1349	test_ioctl_ioas_alloc(&ioas_id);
	1350	vfio_ioas_cmd.ioas_id = ioas_id;
	1351	ASSERT_EQ(0, ioctl(self->fd, IOMMU_VFIO_IOAS, &vfio_ioas_cmd));
	1352
	1353	/* Same id comes back from get */
	1354	vfio_ioas_cmd.op = IOMMU_VFIO_IOAS_GET;
	1355	ASSERT_EQ(0, ioctl(self->fd, IOMMU_VFIO_IOAS, &vfio_ioas_cmd));
	1356	ASSERT_EQ(ioas_id, vfio_ioas_cmd.ioas_id);
	1357
	1358	/* Clear works */
	1359	vfio_ioas_cmd.op = IOMMU_VFIO_IOAS_CLEAR;
	1360	ASSERT_EQ(0, ioctl(self->fd, IOMMU_VFIO_IOAS, &vfio_ioas_cmd));
	1361	vfio_ioas_cmd.op = IOMMU_VFIO_IOAS_GET;
	1362	EXPECT_ERRNO(ENODEV, ioctl(self->fd, IOMMU_VFIO_IOAS, &vfio_ioas_cmd));
	1363	}
	1364
	1365	FIXTURE(vfio_compat_mock_domain)
	1366	{
	1367	int fd;
	1368	uint32_t ioas_id;
	1369	};
	1370
	1371	FIXTURE_VARIANT(vfio_compat_mock_domain)
	1372	{
	1373	unsigned int version;
	1374	};
	1375
	1376	FIXTURE_SETUP(vfio_compat_mock_domain)
	1377	{
	1378	struct iommu_vfio_ioas vfio_ioas_cmd = {
	1379	.size = sizeof(vfio_ioas_cmd),
	1380	.op = IOMMU_VFIO_IOAS_SET,
	1381	};
	1382
	1383	self->fd = open("/dev/iommu", O_RDWR);
	1384	ASSERT_NE(-1, self->fd);
	1385
	1386	/* Create what VFIO would consider a group */
	1387	test_ioctl_ioas_alloc(&self->ioas_id);
	1388	test_cmd_mock_domain(self->ioas_id, NULL, NULL);
	1389
	1390	/* Attach it to the vfio compat */
	1391	vfio_ioas_cmd.ioas_id = self->ioas_id;
	1392	ASSERT_EQ(0, ioctl(self->fd, IOMMU_VFIO_IOAS, &vfio_ioas_cmd));
	1393	ASSERT_EQ(0, ioctl(self->fd, VFIO_SET_IOMMU, variant->version));
	1394	}
	1395
	1396	FIXTURE_TEARDOWN(vfio_compat_mock_domain)
	1397	{
	1398	teardown_iommufd(self->fd, _metadata);
	1399	}
	1400
	1401	FIXTURE_VARIANT_ADD(vfio_compat_mock_domain, Ver1v2)
	1402	{
	1403	.version = VFIO_TYPE1v2_IOMMU,
	1404	};
	1405
	1406	FIXTURE_VARIANT_ADD(vfio_compat_mock_domain, Ver1v0)
	1407	{
	1408	.version = VFIO_TYPE1_IOMMU,
	1409	};
	1410
	1411	TEST_F(vfio_compat_mock_domain, simple_close)
	1412	{
	1413	}
	1414
	1415	TEST_F(vfio_compat_mock_domain, option_huge_pages)
	1416	{
	1417	struct iommu_option cmd = {
	1418	.size = sizeof(cmd),
	1419	.option_id = IOMMU_OPTION_HUGE_PAGES,
	1420	.op = IOMMU_OPTION_OP_GET,
	1421	.val64 = 3,
	1422	.object_id = self->ioas_id,
	1423	};
	1424
	1425	ASSERT_EQ(0, ioctl(self->fd, IOMMU_OPTION, &cmd));
	1426	if (variant->version == VFIO_TYPE1_IOMMU) {
	1427	ASSERT_EQ(0, cmd.val64);
	1428	} else {
	1429	ASSERT_EQ(1, cmd.val64);
	1430	}
	1431	}
	1432
	1433	/*
	1434	* Execute an ioctl command stored in buffer and check that the result does not
	1435	* overflow memory.
	1436	*/
	1437	static bool is_filled(const void *buf, uint8_t c, size_t len)
	1438	{
	1439	const uint8_t *cbuf = buf;
	1440
	1441	for (; len; cbuf++, len--)
	1442	if (*cbuf != c)
	1443	return false;
	1444	return true;
	1445	}
	1446
	1447	#define ioctl_check_buf(fd, cmd) \
	1448	({ \
	1449	size_t _cmd_len = (__u32 )buffer; \
	1450	\
	1451	memset(buffer + _cmd_len, 0xAA, BUFFER_SIZE - _cmd_len); \
	1452	ASSERT_EQ(0, ioctl(fd, cmd, buffer)); \
	1453	ASSERT_EQ(true, is_filled(buffer + _cmd_len, 0xAA, \
	1454	BUFFER_SIZE - _cmd_len)); \
	1455	})
	1456
	1457	static void check_vfio_info_cap_chain(struct __test_metadata *_metadata,
	1458	struct vfio_iommu_type1_info *info_cmd)
	1459	{
	1460	const struct vfio_info_cap_header *cap;
	1461
	1462	ASSERT_GE(info_cmd->argsz, info_cmd->cap_offset + sizeof(*cap));
	1463	cap = buffer + info_cmd->cap_offset;
	1464	while (true) {
	1465	size_t cap_size;
	1466
	1467	if (cap->next)
	1468	cap_size = (buffer + cap->next) - (void *)cap;
	1469	else
	1470	cap_size = (buffer + info_cmd->argsz) - (void *)cap;
	1471
	1472	switch (cap->id) {
	1473	case VFIO_IOMMU_TYPE1_INFO_CAP_IOVA_RANGE: {
	1474	struct vfio_iommu_type1_info_cap_iova_range *data =
	1475	(void *)cap;
	1476
	1477	ASSERT_EQ(1, data->header.version);
	1478	ASSERT_EQ(1, data->nr_iovas);
	1479	EXPECT_EQ(MOCK_APERTURE_START,
	1480	data->iova_ranges[0].start);
	1481	EXPECT_EQ(MOCK_APERTURE_LAST, data->iova_ranges[0].end);
	1482	break;
	1483	}
	1484	case VFIO_IOMMU_TYPE1_INFO_DMA_AVAIL: {
	1485	struct vfio_iommu_type1_info_dma_avail *data =
	1486	(void *)cap;
	1487
	1488	ASSERT_EQ(1, data->header.version);
	1489	ASSERT_EQ(sizeof(*data), cap_size);
	1490	break;
	1491	}
	1492	default:
	1493	ASSERT_EQ(false, true);
	1494	break;
	1495	}
	1496	if (!cap->next)
	1497	break;
	1498
	1499	ASSERT_GE(info_cmd->argsz, cap->next + sizeof(*cap));
	1500	ASSERT_GE(buffer + cap->next, (void *)cap);
	1501	cap = buffer + cap->next;
	1502	}
	1503	}
	1504
	1505	TEST_F(vfio_compat_mock_domain, get_info)
	1506	{
	1507	struct vfio_iommu_type1_info *info_cmd = buffer;
	1508	unsigned int i;
	1509	size_t caplen;
	1510
	1511	/* Pre-cap ABI */
	1512	*info_cmd = (struct vfio_iommu_type1_info){
	1513	.argsz = offsetof(struct vfio_iommu_type1_info, cap_offset),
	1514	};
	1515	ioctl_check_buf(self->fd, VFIO_IOMMU_GET_INFO);
	1516	ASSERT_NE(0, info_cmd->iova_pgsizes);
	1517	ASSERT_EQ(VFIO_IOMMU_INFO_PGSIZES \| VFIO_IOMMU_INFO_CAPS,
	1518	info_cmd->flags);
	1519
	1520	/* Read the cap chain size */
	1521	*info_cmd = (struct vfio_iommu_type1_info){
	1522	.argsz = sizeof(*info_cmd),
	1523	};
	1524	ioctl_check_buf(self->fd, VFIO_IOMMU_GET_INFO);
	1525	ASSERT_NE(0, info_cmd->iova_pgsizes);
	1526	ASSERT_EQ(VFIO_IOMMU_INFO_PGSIZES \| VFIO_IOMMU_INFO_CAPS,
	1527	info_cmd->flags);
	1528	ASSERT_EQ(0, info_cmd->cap_offset);
	1529	ASSERT_LT(sizeof(*info_cmd), info_cmd->argsz);
	1530
	1531	/* Read the caps, kernel should never create a corrupted caps */
	1532	caplen = info_cmd->argsz;
	1533	for (i = sizeof(*info_cmd); i < caplen; i++) {
	1534	*info_cmd = (struct vfio_iommu_type1_info){
	1535	.argsz = i,
	1536	};
	1537	ioctl_check_buf(self->fd, VFIO_IOMMU_GET_INFO);
	1538	ASSERT_EQ(VFIO_IOMMU_INFO_PGSIZES \| VFIO_IOMMU_INFO_CAPS,
	1539	info_cmd->flags);
	1540	if (!info_cmd->cap_offset)
	1541	continue;
	1542	check_vfio_info_cap_chain(_metadata, info_cmd);
	1543	}
	1544	}
	1545
	1546	static void shuffle_array(unsigned long *array, size_t nelms)
	1547	{
	1548	unsigned int i;
	1549
	1550	/* Shuffle */
	1551	for (i = 0; i != nelms; i++) {
	1552	unsigned long tmp = array[i];
	1553	unsigned int other = rand() % (nelms - i);
	1554
	1555	array[i] = array[other];
	1556	array[other] = tmp;
	1557	}
	1558	}
	1559
	1560	TEST_F(vfio_compat_mock_domain, map)
	1561	{
	1562	struct vfio_iommu_type1_dma_map map_cmd = {
	1563	.argsz = sizeof(map_cmd),
	1564	.flags = VFIO_DMA_MAP_FLAG_READ \| VFIO_DMA_MAP_FLAG_WRITE,
	1565	.vaddr = (uintptr_t)buffer,
	1566	.size = BUFFER_SIZE,
	1567	.iova = MOCK_APERTURE_START,
	1568	};
	1569	struct vfio_iommu_type1_dma_unmap unmap_cmd = {
	1570	.argsz = sizeof(unmap_cmd),
	1571	.size = BUFFER_SIZE,
	1572	.iova = MOCK_APERTURE_START,
	1573	};
	1574	unsigned long pages_iova[BUFFER_SIZE / PAGE_SIZE];
	1575	unsigned int i;
	1576
	1577	/* Simple map/unmap */
	1578	ASSERT_EQ(0, ioctl(self->fd, VFIO_IOMMU_MAP_DMA, &map_cmd));
	1579	ASSERT_EQ(0, ioctl(self->fd, VFIO_IOMMU_UNMAP_DMA, &unmap_cmd));
	1580	ASSERT_EQ(BUFFER_SIZE, unmap_cmd.size);
	1581
	1582	/* UNMAP_FLAG_ALL requres 0 iova/size */
	1583	ASSERT_EQ(0, ioctl(self->fd, VFIO_IOMMU_MAP_DMA, &map_cmd));
	1584	unmap_cmd.flags = VFIO_DMA_UNMAP_FLAG_ALL;
	1585	EXPECT_ERRNO(EINVAL, ioctl(self->fd, VFIO_IOMMU_UNMAP_DMA, &unmap_cmd));
	1586
	1587	unmap_cmd.iova = 0;
	1588	unmap_cmd.size = 0;
	1589	ASSERT_EQ(0, ioctl(self->fd, VFIO_IOMMU_UNMAP_DMA, &unmap_cmd));
	1590	ASSERT_EQ(BUFFER_SIZE, unmap_cmd.size);
	1591
	1592	/* Small pages */
	1593	for (i = 0; i != ARRAY_SIZE(pages_iova); i++) {
	1594	map_cmd.iova = pages_iova[i] =
	1595	MOCK_APERTURE_START + i * PAGE_SIZE;
	1596	map_cmd.vaddr = (uintptr_t)buffer + i * PAGE_SIZE;
	1597	map_cmd.size = PAGE_SIZE;
	1598	ASSERT_EQ(0, ioctl(self->fd, VFIO_IOMMU_MAP_DMA, &map_cmd));
	1599	}
	1600	shuffle_array(pages_iova, ARRAY_SIZE(pages_iova));
	1601
	1602	unmap_cmd.flags = 0;
	1603	unmap_cmd.size = PAGE_SIZE;
	1604	for (i = 0; i != ARRAY_SIZE(pages_iova); i++) {
	1605	unmap_cmd.iova = pages_iova[i];
	1606	ASSERT_EQ(0, ioctl(self->fd, VFIO_IOMMU_UNMAP_DMA, &unmap_cmd));
	1607	}
	1608	}
	1609
	1610	TEST_F(vfio_compat_mock_domain, huge_map)
	1611	{
	1612	size_t buf_size = HUGEPAGE_SIZE * 2;
	1613	struct vfio_iommu_type1_dma_map map_cmd = {
	1614	.argsz = sizeof(map_cmd),
	1615	.flags = VFIO_DMA_MAP_FLAG_READ \| VFIO_DMA_MAP_FLAG_WRITE,
	1616	.size = buf_size,
	1617	.iova = MOCK_APERTURE_START,
	1618	};
	1619	struct vfio_iommu_type1_dma_unmap unmap_cmd = {
	1620	.argsz = sizeof(unmap_cmd),
	1621	};
	1622	unsigned long pages_iova[16];
	1623	unsigned int i;
	1624	void *buf;
	1625
	1626	/* Test huge pages and splitting */
	1627	buf = mmap(0, buf_size, PROT_READ \| PROT_WRITE,
	1628	MAP_SHARED \| MAP_ANONYMOUS \| MAP_HUGETLB \| MAP_POPULATE, -1,
	1629	0);
	1630	ASSERT_NE(MAP_FAILED, buf);
	1631	map_cmd.vaddr = (uintptr_t)buf;
	1632	ASSERT_EQ(0, ioctl(self->fd, VFIO_IOMMU_MAP_DMA, &map_cmd));
	1633
	1634	unmap_cmd.size = buf_size / ARRAY_SIZE(pages_iova);
	1635	for (i = 0; i != ARRAY_SIZE(pages_iova); i++)
	1636	pages_iova[i] = MOCK_APERTURE_START + (i * unmap_cmd.size);
	1637	shuffle_array(pages_iova, ARRAY_SIZE(pages_iova));
	1638
	1639	/* type1 mode can cut up larger mappings, type1v2 always fails */
	1640	for (i = 0; i != ARRAY_SIZE(pages_iova); i++) {
	1641	unmap_cmd.iova = pages_iova[i];
	1642	unmap_cmd.size = buf_size / ARRAY_SIZE(pages_iova);
	1643	if (variant->version == VFIO_TYPE1_IOMMU) {
	1644	ASSERT_EQ(0, ioctl(self->fd, VFIO_IOMMU_UNMAP_DMA,
	1645	&unmap_cmd));
	1646	} else {
	1647	EXPECT_ERRNO(ENOENT,
	1648	ioctl(self->fd, VFIO_IOMMU_UNMAP_DMA,
	1649	&unmap_cmd));
	1650	}
	1651	}
	1652	}
	1653
	1654	TEST_HARNESS_MAIN

tools/testing/selftests/iommu/iommufd_fail_nth.c+580−0

@@ -0,0 +1,580 @@
	1	// SPDX-License-Identifier: GPL-2.0-only
	2	/* Copyright (c) 2021-2022, NVIDIA CORPORATION & AFFILIATES
	3	*
	4	* These tests are "kernel integrity" tests. They are looking for kernel
	5	* WARN/OOPS/kasn/etc splats triggered by kernel sanitizers & debugging
	6	* features. It does not attempt to verify that the system calls are doing what
	7	* they are supposed to do.
	8	*
	9	* The basic philosophy is to run a sequence of calls that will succeed and then
	10	* sweep every failure injection point on that call chain to look for
	11	* interesting things in error handling.
	12	*
	13	* This test is best run with:
	14	* echo 1 > /proc/sys/kernel/panic_on_warn
	15	* If something is actually going wrong.
	16	*/
	17	#include <fcntl.h>
	18	#include <dirent.h>
	19
	20	#define __EXPORTED_HEADERS__
	21	#include <linux/vfio.h>
	22
	23	#include "iommufd_utils.h"
	24
	25	static bool have_fault_injection;
	26
	27	static int writeat(int dfd, const char fn, const char val)
	28	{
	29	size_t val_len = strlen(val);
	30	ssize_t res;
	31	int fd;
	32
	33	fd = openat(dfd, fn, O_WRONLY);
	34	if (fd == -1)
	35	return -1;
	36	res = write(fd, val, val_len);
	37	assert(res == val_len);
	38	close(fd);
	39	return 0;
	40	}
	41
	42	static __attribute__((constructor)) void setup_buffer(void)
	43	{
	44	BUFFER_SIZE = 210241024;
	45
	46	buffer = mmap(0, BUFFER_SIZE, PROT_READ \| PROT_WRITE,
	47	MAP_SHARED \| MAP_ANONYMOUS, -1, 0);
	48	}
	49
	50	/*
	51	* This sets up fail_injection in a way that is useful for this test.
	52	* It does not attempt to restore things back to how they were.
	53	*/
	54	static __attribute__((constructor)) void setup_fault_injection(void)
	55	{
	56	DIR *debugfs = opendir("/sys/kernel/debug/");
	57	struct dirent *dent;
	58
	59	if (!debugfs)
	60	return;
	61
	62	/* Allow any allocation call to be fault injected */
	63	if (writeat(dirfd(debugfs), "failslab/ignore-gfp-wait", "N"))
	64	return;
	65	writeat(dirfd(debugfs), "fail_page_alloc/ignore-gfp-wait", "N");
	66	writeat(dirfd(debugfs), "fail_page_alloc/ignore-gfp-highmem", "N");
	67
	68	while ((dent = readdir(debugfs))) {
	69	char fn[300];
	70
	71	if (strncmp(dent->d_name, "fail", 4) != 0)
	72	continue;
	73
	74	/* We are looking for kernel splats, quiet down the log */
	75	snprintf(fn, sizeof(fn), "%s/verbose", dent->d_name);
	76	writeat(dirfd(debugfs), fn, "0");
	77	}
	78	closedir(debugfs);
	79	have_fault_injection = true;
	80	}
	81
	82	struct fail_nth_state {
	83	int proc_fd;
	84	unsigned int iteration;
	85	};
	86
	87	static void fail_nth_first(struct __test_metadata *_metadata,
	88	struct fail_nth_state *nth_state)
	89	{
	90	char buf[300];
	91
	92	snprintf(buf, sizeof(buf), "/proc/self/task/%u/fail-nth", getpid());
	93	nth_state->proc_fd = open(buf, O_RDWR);
	94	ASSERT_NE(-1, nth_state->proc_fd);
	95	}
	96
	97	static bool fail_nth_next(struct __test_metadata *_metadata,
	98	struct fail_nth_state *nth_state,
	99	int test_result)
	100	{
	101	static const char disable_nth[] = "0";
	102	char buf[300];
	103
	104	/*
	105	* This is just an arbitrary limit based on the current kernel
	106	* situation. Changes in the kernel can dramtically change the number of
	107	* required fault injection sites, so if this hits it doesn't
	108	* necessarily mean a test failure, just that the limit has to be made
	109	* bigger.
	110	*/
	111	ASSERT_GT(400, nth_state->iteration);
	112	if (nth_state->iteration != 0) {
	113	ssize_t res;
	114	ssize_t res2;
	115
	116	buf[0] = 0;
	117	/*
	118	* Annoyingly disabling the nth can also fail. This means
	119	* the test passed without triggering failure
	120	*/
	121	res = pread(nth_state->proc_fd, buf, sizeof(buf), 0);
	122	if (res == -1 && errno == EFAULT) {
	123	buf[0] = '1';
	124	buf[1] = '\n';
	125	res = 2;
	126	}
	127
	128	res2 = pwrite(nth_state->proc_fd, disable_nth,
	129	ARRAY_SIZE(disable_nth) - 1, 0);
	130	if (res2 == -1 && errno == EFAULT) {
	131	res2 = pwrite(nth_state->proc_fd, disable_nth,
	132	ARRAY_SIZE(disable_nth) - 1, 0);
	133	buf[0] = '1';
	134	buf[1] = '\n';
	135	}
	136	ASSERT_EQ(ARRAY_SIZE(disable_nth) - 1, res2);
	137
	138	/* printf(" nth %u result=%d nth=%u\n", nth_state->iteration,
	139	test_result, atoi(buf)); */
	140	fflush(stdout);
	141	ASSERT_LT(1, res);
	142	if (res != 2 \|\| buf[0] != '0' \|\| buf[1] != '\n')
	143	return false;
	144	} else {
	145	/* printf(" nth %u result=%d\n", nth_state->iteration,
	146	test_result); */
	147	}
	148	nth_state->iteration++;
	149	return true;
	150	}
	151
	152	/*
	153	* This is called during the test to start failure injection. It allows the test
	154	* to do some setup that has already been swept and thus reduce the required
	155	* iterations.
	156	*/
	157	void __fail_nth_enable(struct __test_metadata *_metadata,
	158	struct fail_nth_state *nth_state)
	159	{
	160	char buf[300];
	161	size_t len;
	162
	163	if (!nth_state->iteration)
	164	return;
	165
	166	len = snprintf(buf, sizeof(buf), "%u", nth_state->iteration);
	167	ASSERT_EQ(len, pwrite(nth_state->proc_fd, buf, len, 0));
	168	}
	169	#define fail_nth_enable() __fail_nth_enable(_metadata, _nth_state)
	170
	171	#define TEST_FAIL_NTH(fixture_name, name) \
	172	static int test_nth_##name(struct __test_metadata *_metadata, \
	173	FIXTURE_DATA(fixture_name) *self, \
	174	const FIXTURE_VARIANT(fixture_name) \
	175	*variant, \
	176	struct fail_nth_state *_nth_state); \
	177	TEST_F(fixture_name, name) \
	178	{ \
	179	struct fail_nth_state nth_state = {}; \
	180	int test_result = 0; \
	181	\
	182	if (!have_fault_injection) \
	183	SKIP(return, \
	184	"fault injection is not enabled in the kernel"); \
	185	fail_nth_first(_metadata, &nth_state); \
	186	ASSERT_EQ(0, test_nth_##name(_metadata, self, variant, \
	187	&nth_state)); \
	188	while (fail_nth_next(_metadata, &nth_state, test_result)) { \
	189	fixture_name##_teardown(_metadata, self, variant); \
	190	fixture_name##_setup(_metadata, self, variant); \
	191	test_result = test_nth_##name(_metadata, self, \
	192	variant, &nth_state); \
	193	}; \
	194	ASSERT_EQ(0, test_result); \
	195	} \
	196	static int test_nth_##name( \
	197	struct __test_metadata __attribute__((unused)) *_metadata, \
	198	FIXTURE_DATA(fixture_name) __attribute__((unused)) *self, \
	199	const FIXTURE_VARIANT(fixture_name) __attribute__((unused)) \
	200	*variant, \
	201	struct fail_nth_state *_nth_state)
	202
	203	FIXTURE(basic_fail_nth)
	204	{
	205	int fd;
	206	uint32_t access_id;
	207	};
	208
	209	FIXTURE_SETUP(basic_fail_nth)
	210	{
	211	self->fd = -1;
	212	self->access_id = 0;
	213	}
	214
	215	FIXTURE_TEARDOWN(basic_fail_nth)
	216	{
	217	int rc;
	218
	219	if (self->access_id) {
	220	/* The access FD holds the iommufd open until it closes */
	221	rc = _test_cmd_destroy_access(self->access_id);
	222	assert(rc == 0);
	223	}
	224	teardown_iommufd(self->fd, _metadata);
	225	}
	226
	227	/* Cover ioas.c */
	228	TEST_FAIL_NTH(basic_fail_nth, basic)
	229	{
	230	struct iommu_iova_range ranges[10];
	231	uint32_t ioas_id;
	232	__u64 iova;
	233
	234	fail_nth_enable();
	235
	236	self->fd = open("/dev/iommu", O_RDWR);
	237	if (self->fd == -1)
	238	return -1;
	239
	240	if (_test_ioctl_ioas_alloc(self->fd, &ioas_id))
	241	return -1;
	242
	243	{
	244	struct iommu_ioas_iova_ranges ranges_cmd = {
	245	.size = sizeof(ranges_cmd),
	246	.num_iovas = ARRAY_SIZE(ranges),
	247	.ioas_id = ioas_id,
	248	.allowed_iovas = (uintptr_t)ranges,
	249	};
	250	if (ioctl(self->fd, IOMMU_IOAS_IOVA_RANGES, &ranges_cmd))
	251	return -1;
	252	}
	253
	254	{
	255	struct iommu_ioas_allow_iovas allow_cmd = {
	256	.size = sizeof(allow_cmd),
	257	.ioas_id = ioas_id,
	258	.num_iovas = 1,
	259	.allowed_iovas = (uintptr_t)ranges,
	260	};
	261
	262	ranges[0].start = 16*1024;
	263	ranges[0].last = BUFFER_SIZE + 16 * 1024 * 600 - 1;
	264	if (ioctl(self->fd, IOMMU_IOAS_ALLOW_IOVAS, &allow_cmd))
	265	return -1;
	266	}
	267
	268	if (_test_ioctl_ioas_map(self->fd, ioas_id, buffer, BUFFER_SIZE, &iova,
	269	IOMMU_IOAS_MAP_WRITEABLE \|
	270	IOMMU_IOAS_MAP_READABLE))
	271	return -1;
	272
	273	{
	274	struct iommu_ioas_copy copy_cmd = {
	275	.size = sizeof(copy_cmd),
	276	.flags = IOMMU_IOAS_MAP_WRITEABLE \|
	277	IOMMU_IOAS_MAP_READABLE,
	278	.dst_ioas_id = ioas_id,
	279	.src_ioas_id = ioas_id,
	280	.src_iova = iova,
	281	.length = sizeof(ranges),
	282	};
	283
	284	if (ioctl(self->fd, IOMMU_IOAS_COPY, &copy_cmd))
	285	return -1;
	286	}
	287
	288	if (_test_ioctl_ioas_unmap(self->fd, ioas_id, iova, BUFFER_SIZE,
	289	NULL))
	290	return -1;
	291	/* Failure path of no IOVA to unmap */
	292	_test_ioctl_ioas_unmap(self->fd, ioas_id, iova, BUFFER_SIZE, NULL);
	293	return 0;
	294	}
	295
	296	/* iopt_area_fill_domains() and iopt_area_fill_domain() */
	297	TEST_FAIL_NTH(basic_fail_nth, map_domain)
	298	{
	299	uint32_t ioas_id;
	300	__u32 device_id;
	301	__u32 hwpt_id;
	302	__u64 iova;
	303
	304	self->fd = open("/dev/iommu", O_RDWR);
	305	if (self->fd == -1)
	306	return -1;
	307
	308	if (_test_ioctl_ioas_alloc(self->fd, &ioas_id))
	309	return -1;
	310
	311	if (_test_ioctl_set_temp_memory_limit(self->fd, 32))
	312	return -1;
	313
	314	fail_nth_enable();
	315
	316	if (_test_cmd_mock_domain(self->fd, ioas_id, &device_id, &hwpt_id))
	317	return -1;
	318
	319	if (_test_ioctl_ioas_map(self->fd, ioas_id, buffer, 262144, &iova,
	320	IOMMU_IOAS_MAP_WRITEABLE \|
	321	IOMMU_IOAS_MAP_READABLE))
	322	return -1;
	323
	324	if (_test_ioctl_destroy(self->fd, device_id))
	325	return -1;
	326	if (_test_ioctl_destroy(self->fd, hwpt_id))
	327	return -1;
	328
	329	if (_test_cmd_mock_domain(self->fd, ioas_id, &device_id, &hwpt_id))
	330	return -1;
	331	return 0;
	332	}
	333
	334	TEST_FAIL_NTH(basic_fail_nth, map_two_domains)
	335	{
	336	uint32_t ioas_id;
	337	__u32 device_id2;
	338	__u32 device_id;
	339	__u32 hwpt_id2;
	340	__u32 hwpt_id;
	341	__u64 iova;
	342
	343	self->fd = open("/dev/iommu", O_RDWR);
	344	if (self->fd == -1)
	345	return -1;
	346
	347	if (_test_ioctl_ioas_alloc(self->fd, &ioas_id))
	348	return -1;
	349
	350	if (_test_ioctl_set_temp_memory_limit(self->fd, 32))
	351	return -1;
	352
	353	if (_test_cmd_mock_domain(self->fd, ioas_id, &device_id, &hwpt_id))
	354	return -1;
	355
	356	fail_nth_enable();
	357
	358	if (_test_cmd_mock_domain(self->fd, ioas_id, &device_id2, &hwpt_id2))
	359	return -1;
	360
	361	if (_test_ioctl_ioas_map(self->fd, ioas_id, buffer, 262144, &iova,
	362	IOMMU_IOAS_MAP_WRITEABLE \|
	363	IOMMU_IOAS_MAP_READABLE))
	364	return -1;
	365
	366	if (_test_ioctl_destroy(self->fd, device_id))
	367	return -1;
	368	if (_test_ioctl_destroy(self->fd, hwpt_id))
	369	return -1;
	370
	371	if (_test_ioctl_destroy(self->fd, device_id2))
	372	return -1;
	373	if (_test_ioctl_destroy(self->fd, hwpt_id2))
	374	return -1;
	375
	376	if (_test_cmd_mock_domain(self->fd, ioas_id, &device_id, &hwpt_id))
	377	return -1;
	378	if (_test_cmd_mock_domain(self->fd, ioas_id, &device_id2, &hwpt_id2))
	379	return -1;
	380	return 0;
	381	}
	382
	383	TEST_FAIL_NTH(basic_fail_nth, access_rw)
	384	{
	385	uint64_t tmp_big[4096];
	386	uint32_t ioas_id;
	387	uint16_t tmp[32];
	388	__u64 iova;
	389
	390	self->fd = open("/dev/iommu", O_RDWR);
	391	if (self->fd == -1)
	392	return -1;
	393
	394	if (_test_ioctl_ioas_alloc(self->fd, &ioas_id))
	395	return -1;
	396
	397	if (_test_ioctl_set_temp_memory_limit(self->fd, 32))
	398	return -1;
	399
	400	if (_test_ioctl_ioas_map(self->fd, ioas_id, buffer, 262144, &iova,
	401	IOMMU_IOAS_MAP_WRITEABLE \|
	402	IOMMU_IOAS_MAP_READABLE))
	403	return -1;
	404
	405	fail_nth_enable();
	406
	407	if (_test_cmd_create_access(self->fd, ioas_id, &self->access_id, 0))
	408	return -1;
	409
	410	{
	411	struct iommu_test_cmd access_cmd = {
	412	.size = sizeof(access_cmd),
	413	.op = IOMMU_TEST_OP_ACCESS_RW,
	414	.id = self->access_id,
	415	.access_rw = { .iova = iova,
	416	.length = sizeof(tmp),
	417	.uptr = (uintptr_t)tmp },
	418	};
	419
	420	// READ
	421	if (ioctl(self->fd, _IOMMU_TEST_CMD(IOMMU_TEST_OP_ACCESS_RW),
	422	&access_cmd))
	423	return -1;
	424
	425	access_cmd.access_rw.flags = MOCK_ACCESS_RW_WRITE;
	426	if (ioctl(self->fd, _IOMMU_TEST_CMD(IOMMU_TEST_OP_ACCESS_RW),
	427	&access_cmd))
	428	return -1;
	429
	430	access_cmd.access_rw.flags = MOCK_ACCESS_RW_SLOW_PATH;
	431	if (ioctl(self->fd, _IOMMU_TEST_CMD(IOMMU_TEST_OP_ACCESS_RW),
	432	&access_cmd))
	433	return -1;
	434	access_cmd.access_rw.flags = MOCK_ACCESS_RW_SLOW_PATH \|
	435	MOCK_ACCESS_RW_WRITE;
	436	if (ioctl(self->fd, _IOMMU_TEST_CMD(IOMMU_TEST_OP_ACCESS_RW),
	437	&access_cmd))
	438	return -1;
	439	}
	440
	441	{
	442	struct iommu_test_cmd access_cmd = {
	443	.size = sizeof(access_cmd),
	444	.op = IOMMU_TEST_OP_ACCESS_RW,
	445	.id = self->access_id,
	446	.access_rw = { .iova = iova,
	447	.flags = MOCK_ACCESS_RW_SLOW_PATH,
	448	.length = sizeof(tmp_big),
	449	.uptr = (uintptr_t)tmp_big },
	450	};
	451
	452	if (ioctl(self->fd, _IOMMU_TEST_CMD(IOMMU_TEST_OP_ACCESS_RW),
	453	&access_cmd))
	454	return -1;
	455	}
	456	if (_test_cmd_destroy_access(self->access_id))
	457	return -1;
	458	self->access_id = 0;
	459	return 0;
	460	}
	461
	462	/* pages.c access functions */
	463	TEST_FAIL_NTH(basic_fail_nth, access_pin)
	464	{
	465	uint32_t access_pages_id;
	466	uint32_t ioas_id;
	467	__u64 iova;
	468
	469	self->fd = open("/dev/iommu", O_RDWR);
	470	if (self->fd == -1)
	471	return -1;
	472
	473	if (_test_ioctl_ioas_alloc(self->fd, &ioas_id))
	474	return -1;
	475
	476	if (_test_ioctl_set_temp_memory_limit(self->fd, 32))
	477	return -1;
	478
	479	if (_test_ioctl_ioas_map(self->fd, ioas_id, buffer, BUFFER_SIZE, &iova,
	480	IOMMU_IOAS_MAP_WRITEABLE \|
	481	IOMMU_IOAS_MAP_READABLE))
	482	return -1;
	483
	484	if (_test_cmd_create_access(self->fd, ioas_id, &self->access_id,
	485	MOCK_FLAGS_ACCESS_CREATE_NEEDS_PIN_PAGES))
	486	return -1;
	487
	488	fail_nth_enable();
	489
	490	{
	491	struct iommu_test_cmd access_cmd = {
	492	.size = sizeof(access_cmd),
	493	.op = IOMMU_TEST_OP_ACCESS_PAGES,
	494	.id = self->access_id,
	495	.access_pages = { .iova = iova,
	496	.length = BUFFER_SIZE,
	497	.uptr = (uintptr_t)buffer },
	498	};
	499
	500	if (ioctl(self->fd, _IOMMU_TEST_CMD(IOMMU_TEST_OP_ACCESS_RW),
	501	&access_cmd))
	502	return -1;
	503	access_pages_id = access_cmd.access_pages.out_access_pages_id;
	504	}
	505
	506	if (_test_cmd_destroy_access_pages(self->fd, self->access_id,
	507	access_pages_id))
	508	return -1;
	509
	510	if (_test_cmd_destroy_access(self->access_id))
	511	return -1;
	512	self->access_id = 0;
	513	return 0;
	514	}
	515
	516	/* iopt_pages_fill_xarray() */
	517	TEST_FAIL_NTH(basic_fail_nth, access_pin_domain)
	518	{
	519	uint32_t access_pages_id;
	520	uint32_t ioas_id;
	521	__u32 device_id;
	522	__u32 hwpt_id;
	523	__u64 iova;
	524
	525	self->fd = open("/dev/iommu", O_RDWR);
	526	if (self->fd == -1)
	527	return -1;
	528
	529	if (_test_ioctl_ioas_alloc(self->fd, &ioas_id))
	530	return -1;
	531
	532	if (_test_ioctl_set_temp_memory_limit(self->fd, 32))
	533	return -1;
	534
	535	if (_test_cmd_mock_domain(self->fd, ioas_id, &device_id, &hwpt_id))
	536	return -1;
	537
	538	if (_test_ioctl_ioas_map(self->fd, ioas_id, buffer, BUFFER_SIZE, &iova,
	539	IOMMU_IOAS_MAP_WRITEABLE \|
	540	IOMMU_IOAS_MAP_READABLE))
	541	return -1;
	542
	543	if (_test_cmd_create_access(self->fd, ioas_id, &self->access_id,
	544	MOCK_FLAGS_ACCESS_CREATE_NEEDS_PIN_PAGES))
	545	return -1;
	546
	547	fail_nth_enable();
	548
	549	{
	550	struct iommu_test_cmd access_cmd = {
	551	.size = sizeof(access_cmd),
	552	.op = IOMMU_TEST_OP_ACCESS_PAGES,
	553	.id = self->access_id,
	554	.access_pages = { .iova = iova,
	555	.length = BUFFER_SIZE,
	556	.uptr = (uintptr_t)buffer },
	557	};
	558
	559	if (ioctl(self->fd, _IOMMU_TEST_CMD(IOMMU_TEST_OP_ACCESS_RW),
	560	&access_cmd))
	561	return -1;
	562	access_pages_id = access_cmd.access_pages.out_access_pages_id;
	563	}
	564
	565	if (_test_cmd_destroy_access_pages(self->fd, self->access_id,
	566	access_pages_id))
	567	return -1;
	568
	569	if (_test_cmd_destroy_access(self->access_id))
	570	return -1;
	571	self->access_id = 0;
	572
	573	if (_test_ioctl_destroy(self->fd, device_id))
	574	return -1;
	575	if (_test_ioctl_destroy(self->fd, hwpt_id))
	576	return -1;
	577	return 0;
	578	}
	579
	580	TEST_HARNESS_MAIN

tools/testing/selftests/iommu/iommufd_utils.h+278−0

@@ -0,0 +1,278 @@
	1	/* SPDX-License-Identifier: GPL-2.0-only */
	2	/* Copyright (c) 2021-2022, NVIDIA CORPORATION & AFFILIATES */
	3	#ifndef __SELFTEST_IOMMUFD_UTILS
	4	#define __SELFTEST_IOMMUFD_UTILS
	5
	6	#include <unistd.h>
	7	#include <stddef.h>
	8	#include <sys/fcntl.h>
	9	#include <sys/ioctl.h>
	10	#include <stdint.h>
	11	#include <assert.h>
	12
	13	#include "../kselftest_harness.h"
	14	#include "../../../../drivers/iommu/iommufd/iommufd_test.h"
	15
	16	/* Hack to make assertions more readable */
	17	#define _IOMMU_TEST_CMD(x) IOMMU_TEST_CMD
	18
	19	static void *buffer;
	20	static unsigned long BUFFER_SIZE;
	21
	22	/*
	23	* Have the kernel check the refcount on pages. I don't know why a freshly
	24	* mmap'd anon non-compound page starts out with a ref of 3
	25	*/
	26	#define check_refs(_ptr, _length, _refs) \
	27	({ \
	28	struct iommu_test_cmd test_cmd = { \
	29	.size = sizeof(test_cmd), \
	30	.op = IOMMU_TEST_OP_MD_CHECK_REFS, \
	31	.check_refs = { .length = _length, \
	32	.uptr = (uintptr_t)(_ptr), \
	33	.refs = _refs }, \
	34	}; \
	35	ASSERT_EQ(0, \
	36	ioctl(self->fd, \
	37	_IOMMU_TEST_CMD(IOMMU_TEST_OP_MD_CHECK_REFS), \
	38	&test_cmd)); \
	39	})
	40
	41	static int _test_cmd_mock_domain(int fd, unsigned int ioas_id, __u32 *device_id,
	42	__u32 *hwpt_id)
	43	{
	44	struct iommu_test_cmd cmd = {
	45	.size = sizeof(cmd),
	46	.op = IOMMU_TEST_OP_MOCK_DOMAIN,
	47	.id = ioas_id,
	48	.mock_domain = {},
	49	};
	50	int ret;
	51
	52	ret = ioctl(fd, IOMMU_TEST_CMD, &cmd);
	53	if (ret)
	54	return ret;
	55	if (device_id)
	56	*device_id = cmd.mock_domain.out_device_id;
	57	assert(cmd.id != 0);
	58	if (hwpt_id)
	59	*hwpt_id = cmd.mock_domain.out_hwpt_id;
	60	return 0;
	61	}
	62	#define test_cmd_mock_domain(ioas_id, device_id, hwpt_id) \
	63	ASSERT_EQ(0, _test_cmd_mock_domain(self->fd, ioas_id, device_id, \
	64	hwpt_id))
	65	#define test_err_mock_domain(_errno, ioas_id, device_id, hwpt_id) \
	66	EXPECT_ERRNO(_errno, _test_cmd_mock_domain(self->fd, ioas_id, \
	67	device_id, hwpt_id))
	68
	69	static int _test_cmd_create_access(int fd, unsigned int ioas_id,
	70	__u32 *access_id, unsigned int flags)
	71	{
	72	struct iommu_test_cmd cmd = {
	73	.size = sizeof(cmd),
	74	.op = IOMMU_TEST_OP_CREATE_ACCESS,
	75	.id = ioas_id,
	76	.create_access = { .flags = flags },
	77	};
	78	int ret;
	79
	80	ret = ioctl(fd, IOMMU_TEST_CMD, &cmd);
	81	if (ret)
	82	return ret;
	83	*access_id = cmd.create_access.out_access_fd;
	84	return 0;
	85	}
	86	#define test_cmd_create_access(ioas_id, access_id, flags) \
	87	ASSERT_EQ(0, _test_cmd_create_access(self->fd, ioas_id, access_id, \
	88	flags))
	89
	90	static int _test_cmd_destroy_access(unsigned int access_id)
	91	{
	92	return close(access_id);
	93	}
	94	#define test_cmd_destroy_access(access_id) \
	95	ASSERT_EQ(0, _test_cmd_destroy_access(access_id))
	96
	97	static int _test_cmd_destroy_access_pages(int fd, unsigned int access_id,
	98	unsigned int access_pages_id)
	99	{
	100	struct iommu_test_cmd cmd = {
	101	.size = sizeof(cmd),
	102	.op = IOMMU_TEST_OP_DESTROY_ACCESS_PAGES,
	103	.id = access_id,
	104	.destroy_access_pages = { .access_pages_id = access_pages_id },
	105	};
	106	return ioctl(fd, IOMMU_TEST_CMD, &cmd);
	107	}
	108	#define test_cmd_destroy_access_pages(access_id, access_pages_id) \
	109	ASSERT_EQ(0, _test_cmd_destroy_access_pages(self->fd, access_id, \
	110	access_pages_id))
	111	#define test_err_destroy_access_pages(_errno, access_id, access_pages_id) \
	112	EXPECT_ERRNO(_errno, _test_cmd_destroy_access_pages( \
	113	self->fd, access_id, access_pages_id))
	114
	115	static int _test_ioctl_destroy(int fd, unsigned int id)
	116	{
	117	struct iommu_destroy cmd = {
	118	.size = sizeof(cmd),
	119	.id = id,
	120	};
	121	return ioctl(fd, IOMMU_DESTROY, &cmd);
	122	}
	123	#define test_ioctl_destroy(id) ASSERT_EQ(0, _test_ioctl_destroy(self->fd, id))
	124
	125	static int _test_ioctl_ioas_alloc(int fd, __u32 *id)
	126	{
	127	struct iommu_ioas_alloc cmd = {
	128	.size = sizeof(cmd),
	129	};
	130	int ret;
	131
	132	ret = ioctl(fd, IOMMU_IOAS_ALLOC, &cmd);
	133	if (ret)
	134	return ret;
	135	*id = cmd.out_ioas_id;
	136	return 0;
	137	}
	138	#define test_ioctl_ioas_alloc(id) \
	139	({ \
	140	ASSERT_EQ(0, _test_ioctl_ioas_alloc(self->fd, id)); \
	141	ASSERT_NE(0, *(id)); \
	142	})
	143
	144	static int _test_ioctl_ioas_map(int fd, unsigned int ioas_id, void *buffer,
	145	size_t length, __u64 *iova, unsigned int flags)
	146	{
	147	struct iommu_ioas_map cmd = {
	148	.size = sizeof(cmd),
	149	.flags = flags,
	150	.ioas_id = ioas_id,
	151	.user_va = (uintptr_t)buffer,
	152	.length = length,
	153	};
	154	int ret;
	155
	156	if (flags & IOMMU_IOAS_MAP_FIXED_IOVA)
	157	cmd.iova = *iova;
	158
	159	ret = ioctl(fd, IOMMU_IOAS_MAP, &cmd);
	160	*iova = cmd.iova;
	161	return ret;
	162	}
	163	#define test_ioctl_ioas_map(buffer, length, iova_p) \
	164	ASSERT_EQ(0, _test_ioctl_ioas_map(self->fd, self->ioas_id, buffer, \
	165	length, iova_p, \
	166	IOMMU_IOAS_MAP_WRITEABLE \| \
	167	IOMMU_IOAS_MAP_READABLE))
	168
	169	#define test_err_ioctl_ioas_map(_errno, buffer, length, iova_p) \
	170	EXPECT_ERRNO(_errno, \
	171	_test_ioctl_ioas_map(self->fd, self->ioas_id, buffer, \
	172	length, iova_p, \
	173	IOMMU_IOAS_MAP_WRITEABLE \| \
	174	IOMMU_IOAS_MAP_READABLE))
	175
	176	#define test_ioctl_ioas_map_id(ioas_id, buffer, length, iova_p) \
	177	ASSERT_EQ(0, _test_ioctl_ioas_map(self->fd, ioas_id, buffer, length, \
	178	iova_p, \
	179	IOMMU_IOAS_MAP_WRITEABLE \| \
	180	IOMMU_IOAS_MAP_READABLE))
	181
	182	#define test_ioctl_ioas_map_fixed(buffer, length, iova) \
	183	({ \
	184	__u64 __iova = iova; \
	185	ASSERT_EQ(0, _test_ioctl_ioas_map( \
	186	self->fd, self->ioas_id, buffer, length, \
	187	&__iova, \
	188	IOMMU_IOAS_MAP_FIXED_IOVA \| \
	189	IOMMU_IOAS_MAP_WRITEABLE \| \
	190	IOMMU_IOAS_MAP_READABLE)); \
	191	})
	192
	193	#define test_err_ioctl_ioas_map_fixed(_errno, buffer, length, iova) \
	194	({ \
	195	__u64 __iova = iova; \
	196	EXPECT_ERRNO(_errno, \
	197	_test_ioctl_ioas_map( \
	198	self->fd, self->ioas_id, buffer, length, \
	199	&__iova, \
	200	IOMMU_IOAS_MAP_FIXED_IOVA \| \
	201	IOMMU_IOAS_MAP_WRITEABLE \| \
	202	IOMMU_IOAS_MAP_READABLE)); \
	203	})
	204
	205	static int _test_ioctl_ioas_unmap(int fd, unsigned int ioas_id, uint64_t iova,
	206	size_t length, uint64_t *out_len)
	207	{
	208	struct iommu_ioas_unmap cmd = {
	209	.size = sizeof(cmd),
	210	.ioas_id = ioas_id,
	211	.iova = iova,
	212	.length = length,
	213	};
	214	int ret;
	215
	216	ret = ioctl(fd, IOMMU_IOAS_UNMAP, &cmd);
	217	if (out_len)
	218	*out_len = cmd.length;
	219	return ret;
	220	}
	221	#define test_ioctl_ioas_unmap(iova, length) \
	222	ASSERT_EQ(0, _test_ioctl_ioas_unmap(self->fd, self->ioas_id, iova, \
	223	length, NULL))
	224
	225	#define test_ioctl_ioas_unmap_id(ioas_id, iova, length) \
	226	ASSERT_EQ(0, _test_ioctl_ioas_unmap(self->fd, ioas_id, iova, length, \
	227	NULL))
	228
	229	#define test_err_ioctl_ioas_unmap(_errno, iova, length) \
	230	EXPECT_ERRNO(_errno, _test_ioctl_ioas_unmap(self->fd, self->ioas_id, \
	231	iova, length, NULL))
	232
	233	static int _test_ioctl_set_temp_memory_limit(int fd, unsigned int limit)
	234	{
	235	struct iommu_test_cmd memlimit_cmd = {
	236	.size = sizeof(memlimit_cmd),
	237	.op = IOMMU_TEST_OP_SET_TEMP_MEMORY_LIMIT,
	238	.memory_limit = { .limit = limit },
	239	};
	240
	241	return ioctl(fd, _IOMMU_TEST_CMD(IOMMU_TEST_OP_SET_TEMP_MEMORY_LIMIT),
	242	&memlimit_cmd);
	243	}
	244
	245	#define test_ioctl_set_temp_memory_limit(limit) \
	246	ASSERT_EQ(0, _test_ioctl_set_temp_memory_limit(self->fd, limit))
	247
	248	#define test_ioctl_set_default_memory_limit() \
	249	test_ioctl_set_temp_memory_limit(65536)
	250
	251	static void teardown_iommufd(int fd, struct __test_metadata *_metadata)
	252	{
	253	struct iommu_test_cmd test_cmd = {
	254	.size = sizeof(test_cmd),
	255	.op = IOMMU_TEST_OP_MD_CHECK_REFS,
	256	.check_refs = { .length = BUFFER_SIZE,
	257	.uptr = (uintptr_t)buffer },
	258	};
	259
	260	if (fd == -1)
	261	return;
	262
	263	EXPECT_EQ(0, close(fd));
	264
	265	fd = open("/dev/iommu", O_RDWR);
	266	EXPECT_NE(-1, fd);
	267	EXPECT_EQ(0, ioctl(fd, _IOMMU_TEST_CMD(IOMMU_TEST_OP_MD_CHECK_REFS),
	268	&test_cmd));
	269	EXPECT_EQ(0, close(fd));
	270	}
	271
	272	#define EXPECT_ERRNO(expected_errno, cmd) \
	273	({ \
	274	ASSERT_EQ(-1, cmd); \
	275	EXPECT_EQ(expected_errno, errno); \
	276	})
	277
	278	#endif