Merge tag 'for-linus-iommufd' of git://git.kernel.org/pub/scm/linux/kernel/git/jgg/iommufd

Pull iommufd updates from Jason Gunthorpe: "On top of the vfio updates is built some new iommufd functionality: - IOMMU_HWPT_ALLOC allows userspace to directly create the low level IO Page table objects and affiliate them with IOAS objects that hold the translation mapping. This is the basic functionality for the normal IOMMU_DOMAIN_PAGING domains. - VFIO_DEVICE_ATTACH_IOMMUFD_PT can be used to replace the current translation. This is wired up to through all the layers down to the driver so the driver has the ability to implement a hitless replacement. This is necessary to fully support guest behaviors when emulating HW (eg guest atomic change of translation) - IOMMU_GET_HW_INFO returns information about the IOMMU driver HW that owns a VFIO device. This includes support for the Intel iommu, and patches have been posted for all the other server IOMMU. Along the way are a number of internal items: - New iommufd kernel APIs: iommufd_ctx_has_group(), iommufd_device_to_ictx(), iommufd_device_to_id(), iommufd_access_detach(), iommufd_ctx_from_fd(), iommufd_device_replace() - iommufd now internally tracks iommu_groups as it needs some per-group data - Reorganize how the internal hwpt allocation flows to have more robust locking - Improve the access interfaces to support detach and replace of an IOAS from an access - New selftests and a rework of how the selftests creates a mock iommu driver to be more like a real iommu driver" Link: https://lore.kernel.org/lkml/ZO%2FTe6LU1ENf58ZW@nvidia.com/ * tag 'for-linus-iommufd' of git://git.kernel.org/pub/scm/linux/kernel/git/jgg/iommufd: (34 commits) iommufd/selftest: Don't leak the platform device memory when unloading the module iommu/vt-d: Implement hw_info for iommu capability query iommufd/selftest: Add coverage for IOMMU_GET_HW_INFO ioctl iommufd: Add IOMMU_GET_HW_INFO iommu: Add new iommu op to get iommu hardware information iommu: Move dev_iommu_ops() to private header iommufd: Remove iommufd_ref_to_users() iommufd/selftest: Make the mock iommu driver into a real driver vfio: Support IO page table replacement iommufd/selftest: Add IOMMU_TEST_OP_ACCESS_REPLACE_IOAS coverage iommufd: Add iommufd_access_replace() API iommufd: Use iommufd_access_change_ioas in iommufd_access_destroy_object iommufd: Add iommufd_access_change_ioas(_id) helpers iommufd: Allow passing in iopt_access_list_id to iopt_remove_access() vfio: Do not allow !ops->dma_unmap in vfio_pin/unpin_pages() iommufd/selftest: Add a selftest for IOMMU_HWPT_ALLOC iommufd/selftest: Return the real idev id from selftest mock_domain iommufd: Add IOMMU_HWPT_ALLOC iommufd/selftest: Test iommufd_device_replace() iommufd: Make destroy_rwsem use a lock class per object type ...

Linus Torvalds committed 3 years ago

Commit 4debf772 parents ec0e2dc eb501c219 files · +1486 −355Browse files

Unified · Split

19 files

drivers/iommu/intel/iommu.c +19 −0
drivers/iommu/iommu-priv.h +30 −0
drivers/iommu/iommu.c +80 −1
drivers/iommu/iommufd/device.c +552 −179
drivers/iommu/iommufd/hw_pagetable.c +93 −19
drivers/iommu/iommufd/io_pagetable.c +17 −19
drivers/iommu/iommufd/iommufd_private.h +46 −38
drivers/iommu/iommufd/iommufd_test.h +19 −0
drivers/iommu/iommufd/main.c +32 −3
drivers/iommu/iommufd/selftest.c +157 −56
drivers/vfio/iommufd.c +6 −5
drivers/vfio/vfio_main.c +4 −0
include/linux/iommu.h +5 −11
include/linux/iommufd.h +2 −0
include/uapi/linux/iommufd.h +97 −0
include/uapi/linux/vfio.h +6 −0
tools/testing/selftests/iommu/iommufd.c +118 −12
tools/testing/selftests/iommu/iommufd_fail_nth.c +64 −7
tools/testing/selftests/iommu/iommufd_utils.h +139 −5

drivers/iommu/intel/iommu.c+19−0

@@ -22,6 +22,7 @@ · 21 unmodified lines
22	22	#include <linux/spinlock.h>
23	23	#include <linux/syscore_ops.h>
24	24	#include <linux/tboot.h>
	25	+#include <uapi/linux/iommufd.h>
25	26
26	27	#include "iommu.h"
27	28	#include "../dma-iommu.h"
@@ -4732,8 +4733,26 @@ static void intel_iommu_remove_dev_pasid(struct device *dev, ioasid_t pasid) · 4704 unmodified lines
4732	4733	intel_pasid_tear_down_entry(iommu, dev, pasid, false);
4733	4734	}
4734	4735
	4736	+static void intel_iommu_hw_info(struct device dev, u32 length, u32 type)
	4737	+{
	4738	+ struct device_domain_info *info = dev_iommu_priv_get(dev);
	4739	+ struct intel_iommu *iommu = info->iommu;
	4740	+ struct iommu_hw_info_vtd *vtd;
	4741	+
	4742	+ vtd = kzalloc(sizeof(*vtd), GFP_KERNEL);
	4743	+ if (!vtd)
	4744	+ return ERR_PTR(-ENOMEM);
	4745	+
	4746	+ vtd->cap_reg = iommu->cap;
	4747	+ vtd->ecap_reg = iommu->ecap;
	4748	+ length = sizeof(vtd);
	4749	+ *type = IOMMU_HW_INFO_TYPE_INTEL_VTD;
	4750	+ return vtd;
	4751	+}
	4752	+
4735	4753	const struct iommu_ops intel_iommu_ops = {
4736	4754	.capable = intel_iommu_capable,
	4755	+ .hw_info = intel_iommu_hw_info,
4737	4756	.domain_alloc = intel_iommu_domain_alloc,
4738	4757	.probe_device = intel_iommu_probe_device,
4739	4758	.probe_finalize = intel_iommu_probe_finalize,

drivers/iommu/iommu-priv.h+30−0

@@ -0,0 +1,30 @@
	1	+/* SPDX-License-Identifier: GPL-2.0-only */
	2	+/* Copyright (c) 2023, NVIDIA CORPORATION & AFFILIATES.
	3	+ */
	4	+#ifndef __LINUX_IOMMU_PRIV_H
	5	+#define __LINUX_IOMMU_PRIV_H
	6	+
	7	+#include <linux/iommu.h>
	8	+
	9	+static inline const struct iommu_ops dev_iommu_ops(struct device dev)
	10	+{
	11	+ /*
	12	+ * Assume that valid ops must be installed if iommu_probe_device()
	13	+ * has succeeded. The device ops are essentially for internal use
	14	+ * within the IOMMU subsystem itself, so we should be able to trust
	15	+ * ourselves not to misuse the helper.
	16	+ */
	17	+ return dev->iommu->iommu_dev->ops;
	18	+}
	19	+
	20	+int iommu_group_replace_domain(struct iommu_group *group,
	21	+ struct iommu_domain *new_domain);
	22	+
	23	+int iommu_device_register_bus(struct iommu_device *iommu,
	24	+ const struct iommu_ops ops, struct bus_type bus,
	25	+ struct notifier_block *nb);
	26	+void iommu_device_unregister_bus(struct iommu_device *iommu,
	27	+ struct bus_type *bus,
	28	+ struct notifier_block *nb);
	29	+
	30	+#endif /* __LINUX_IOMMU_PRIV_H */

drivers/iommu/iommu.c+80−1

@@ -34,8 +34,10 @@ · 33 unmodified lines
34	34	#include <linux/msi.h>
35	35
36	36	#include "dma-iommu.h"
	37	+#include "iommu-priv.h"
37	38
38	39	#include "iommu-sva.h"
	40	+#include "iommu-priv.h"
39	41
40	42	static struct kset *iommu_group_kset;
41	43	static DEFINE_IDA(iommu_group_ida);
@@ -287,6 +289,48 @@ void iommu_device_unregister(struct iommu_device *iommu) · 245 unmodified lines
287	289	}
288	290	EXPORT_SYMBOL_GPL(iommu_device_unregister);
289	291
	292	+#if IS_ENABLED(CONFIG_IOMMUFD_TEST)
	293	+void iommu_device_unregister_bus(struct iommu_device *iommu,
	294	+ struct bus_type *bus,
	295	+ struct notifier_block *nb)
	296	+{
	297	+ bus_unregister_notifier(bus, nb);
	298	+ iommu_device_unregister(iommu);
	299	+}
	300	+EXPORT_SYMBOL_GPL(iommu_device_unregister_bus);
	301	+
	302	+/*
	303	+ * Register an iommu driver against a single bus. This is only used by iommufd
	304	+ * selftest to create a mock iommu driver. The caller must provide
	305	+ * some memory to hold a notifier_block.
	306	+ */
	307	+int iommu_device_register_bus(struct iommu_device *iommu,
	308	+ const struct iommu_ops ops, struct bus_type bus,
	309	+ struct notifier_block *nb)
	310	+{
	311	+ int err;
	312	+
	313	+ iommu->ops = ops;
	314	+ nb->notifier_call = iommu_bus_notifier;
	315	+ err = bus_register_notifier(bus, nb);
	316	+ if (err)
	317	+ return err;
	318	+
	319	+ spin_lock(&iommu_device_lock);
	320	+ list_add_tail(&iommu->list, &iommu_device_list);
	321	+ spin_unlock(&iommu_device_lock);
	322	+
	323	+ bus->iommu_ops = ops;
	324	+ err = bus_iommu_probe(bus);
	325	+ if (err) {
	326	+ iommu_device_unregister_bus(iommu, bus, nb);
	327	+ return err;
	328	+ }
	329	+ return 0;
	330	+}
	331	+EXPORT_SYMBOL_GPL(iommu_device_register_bus);
	332	+#endif
	333	+
290	334	static struct dev_iommu dev_iommu_get(struct device dev)
291	335	{
292	336	struct dev_iommu *param = dev->iommu;
@@ -2114,6 +2158,32 @@ int iommu_attach_group(struct iommu_domain domain, struct iommu_group group) · 1821 unmodified lines
2114	2158	}
2115	2159	EXPORT_SYMBOL_GPL(iommu_attach_group);
2116	2160
	2161	+/**
	2162	+ * iommu_group_replace_domain - replace the domain that a group is attached to
	2163	+ * @new_domain: new IOMMU domain to replace with
	2164	+ * @group: IOMMU group that will be attached to the new domain
	2165	+ *
	2166	+ * This API allows the group to switch domains without being forced to go to
	2167	+ * the blocking domain in-between.
	2168	+ *
	2169	+ * If the currently attached domain is a core domain (e.g. a default_domain),
	2170	+ * it will act just like the iommu_attach_group().
	2171	+ */
	2172	+int iommu_group_replace_domain(struct iommu_group *group,
	2173	+ struct iommu_domain *new_domain)
	2174	+{
	2175	+ int ret;
	2176	+
	2177	+ if (!new_domain)
	2178	+ return -EINVAL;
	2179	+
	2180	+ mutex_lock(&group->mutex);
	2181	+ ret = __iommu_group_set_domain(group, new_domain);
	2182	+ mutex_unlock(&group->mutex);
	2183	+ return ret;
	2184	+}
	2185	+EXPORT_SYMBOL_NS_GPL(iommu_group_replace_domain, IOMMUFD_INTERNAL);
	2186	+
2117	2187	static int __iommu_device_set_domain(struct iommu_group *group,
2118	2188	struct device *dev,
2119	2189	struct iommu_domain *new_domain,
@@ -2642,6 +2712,14 @@ int iommu_set_pgtable_quirks(struct iommu_domain *domain, · 522 unmodified lines
2642	2712	}
2643	2713	EXPORT_SYMBOL_GPL(iommu_set_pgtable_quirks);
2644	2714
	2715	+/**
	2716	+ * iommu_get_resv_regions - get reserved regions
	2717	+ * @dev: device for which to get reserved regions
	2718	+ * @list: reserved region list for device
	2719	+ *
	2720	+ * This returns a list of reserved IOVA regions specific to this device.
	2721	+ * A domain user should not map IOVA in these ranges.
	2722	+ */
2645	2723	void iommu_get_resv_regions(struct device dev, struct list_head list)
2646	2724	{
2647	2725	const struct iommu_ops *ops = dev_iommu_ops(dev);
@@ -2649,9 +2727,10 @@ void iommu_get_resv_regions(struct device dev, struct list_head list) · 1 unmodified line
2649	2727	if (ops->get_resv_regions)
2650	2728	ops->get_resv_regions(dev, list);
2651	2729	}
	2730	+EXPORT_SYMBOL_GPL(iommu_get_resv_regions);
2652	2731
2653	2732	/**
2654		− * iommu_put_resv_regions - release resered regions
	2733	+ * iommu_put_resv_regions - release reserved regions
2655	2734	* @dev: device for which to free reserved regions
2656	2735	* @list: reserved region list for device
2657	2736	*

drivers/iommu/iommufd/device.c+552−179

@@ -4,6 +4,8 @@ · 3 unmodified lines
4	4	#include <linux/iommufd.h>
5	5	#include <linux/slab.h>
6	6	#include <linux/iommu.h>
	7	+#include <uapi/linux/iommufd.h>
	8	+#include "../iommu-priv.h"
7	9
8	10	#include "io_pagetable.h"
9	11	#include "iommufd_private.h"
@@ -15,13 +17,127 @@ MODULE_PARM_DESC( · 5 unmodified lines
15	17	"Allow IOMMUFD to bind to devices even if the platform cannot isolate "
16	18	"the MSI interrupt window. Enabling this is a security weakness.");
17	19
	20	+static void iommufd_group_release(struct kref *kref)
	21	+{
	22	+ struct iommufd_group *igroup =
	23	+ container_of(kref, struct iommufd_group, ref);
	24	+
	25	+ WARN_ON(igroup->hwpt \|\| !list_empty(&igroup->device_list));
	26	+
	27	+ xa_cmpxchg(&igroup->ictx->groups, iommu_group_id(igroup->group), igroup,
	28	+ NULL, GFP_KERNEL);
	29	+ iommu_group_put(igroup->group);
	30	+ mutex_destroy(&igroup->lock);
	31	+ kfree(igroup);
	32	+}
	33	+
	34	+static void iommufd_put_group(struct iommufd_group *group)
	35	+{
	36	+ kref_put(&group->ref, iommufd_group_release);
	37	+}
	38	+
	39	+static bool iommufd_group_try_get(struct iommufd_group *igroup,
	40	+ struct iommu_group *group)
	41	+{
	42	+ if (!igroup)
	43	+ return false;
	44	+ /*
	45	+ * group ID's cannot be re-used until the group is put back which does
	46	+ * not happen if we could get an igroup pointer under the xa_lock.
	47	+ */
	48	+ if (WARN_ON(igroup->group != group))
	49	+ return false;
	50	+ return kref_get_unless_zero(&igroup->ref);
	51	+}
	52	+
	53	+/*
	54	+ * iommufd needs to store some more data for each iommu_group, we keep a
	55	+ * parallel xarray indexed by iommu_group id to hold this instead of putting it
	56	+ * in the core structure. To keep things simple the iommufd_group memory is
	57	+ * unique within the iommufd_ctx. This makes it easy to check there are no
	58	+ * memory leaks.
	59	+ */
	60	+static struct iommufd_group iommufd_get_group(struct iommufd_ctx ictx,
	61	+ struct device *dev)
	62	+{
	63	+ struct iommufd_group *new_igroup;
	64	+ struct iommufd_group *cur_igroup;
	65	+ struct iommufd_group *igroup;
	66	+ struct iommu_group *group;
	67	+ unsigned int id;
	68	+
	69	+ group = iommu_group_get(dev);
	70	+ if (!group)
	71	+ return ERR_PTR(-ENODEV);
	72	+
	73	+ id = iommu_group_id(group);
	74	+
	75	+ xa_lock(&ictx->groups);
	76	+ igroup = xa_load(&ictx->groups, id);
	77	+ if (iommufd_group_try_get(igroup, group)) {
	78	+ xa_unlock(&ictx->groups);
	79	+ iommu_group_put(group);
	80	+ return igroup;
	81	+ }
	82	+ xa_unlock(&ictx->groups);
	83	+
	84	+ new_igroup = kzalloc(sizeof(*new_igroup), GFP_KERNEL);
	85	+ if (!new_igroup) {
	86	+ iommu_group_put(group);
	87	+ return ERR_PTR(-ENOMEM);
	88	+ }
	89	+
	90	+ kref_init(&new_igroup->ref);
	91	+ mutex_init(&new_igroup->lock);
	92	+ INIT_LIST_HEAD(&new_igroup->device_list);
	93	+ new_igroup->sw_msi_start = PHYS_ADDR_MAX;
	94	+ /* group reference moves into new_igroup */
	95	+ new_igroup->group = group;
	96	+
	97	+ /*
	98	+ * The ictx is not additionally refcounted here becase all objects using
	99	+ * an igroup must put it before their destroy completes.
	100	+ */
	101	+ new_igroup->ictx = ictx;
	102	+
	103	+ /*
	104	+ * We dropped the lock so igroup is invalid. NULL is a safe and likely
	105	+ * value to assume for the xa_cmpxchg algorithm.
	106	+ */
	107	+ cur_igroup = NULL;
	108	+ xa_lock(&ictx->groups);
	109	+ while (true) {
	110	+ igroup = __xa_cmpxchg(&ictx->groups, id, cur_igroup, new_igroup,
	111	+ GFP_KERNEL);
	112	+ if (xa_is_err(igroup)) {
	113	+ xa_unlock(&ictx->groups);
	114	+ iommufd_put_group(new_igroup);
	115	+ return ERR_PTR(xa_err(igroup));
	116	+ }
	117	+
	118	+ /* new_group was successfully installed */
	119	+ if (cur_igroup == igroup) {
	120	+ xa_unlock(&ictx->groups);
	121	+ return new_igroup;
	122	+ }
	123	+
	124	+ /* Check again if the current group is any good */
	125	+ if (iommufd_group_try_get(igroup, group)) {
	126	+ xa_unlock(&ictx->groups);
	127	+ iommufd_put_group(new_igroup);
	128	+ return igroup;
	129	+ }
	130	+ cur_igroup = igroup;
	131	+ }
	132	+}
	133	+
18	134	void iommufd_device_destroy(struct iommufd_object *obj)
19	135	{
20	136	struct iommufd_device *idev =
21	137	container_of(obj, struct iommufd_device, obj);
22	138
23	139	iommu_device_release_dma_owner(idev->dev);
24		− iommu_group_put(idev->group);
	140	+ iommufd_put_group(idev->igroup);
25	141	if (!iommufd_selftest_is_mock_dev(idev->dev))
26	142	iommufd_ctx_put(idev->ictx);
27	143	}
@@ -46,7 +162,7 @@ struct iommufd_device iommufd_device_bind(struct iommufd_ctx ictx, · 18 unmodified lines
46	162	struct device dev, u32 id)
47	163	{
48	164	struct iommufd_device *idev;
49		− struct iommu_group *group;
	165	+ struct iommufd_group *igroup;
50	166	int rc;
51	167
52	168	/*
@@ -56,9 +172,29 @@ struct iommufd_device iommufd_device_bind(struct iommufd_ctx ictx, · 3 unmodified lines
56	172	if (!device_iommu_capable(dev, IOMMU_CAP_CACHE_COHERENCY))
57	173	return ERR_PTR(-EINVAL);
58	174
59		− group = iommu_group_get(dev);
60		− if (!group)
61		− return ERR_PTR(-ENODEV);
	175	+ igroup = iommufd_get_group(ictx, dev);
	176	+ if (IS_ERR(igroup))
	177	+ return ERR_CAST(igroup);
	178	+
	179	+ /*
	180	+ * For historical compat with VFIO the insecure interrupt path is
	181	+ * allowed if the module parameter is set. Secure/Isolated means that a
	182	+ * MemWr operation from the device (eg a simple DMA) cannot trigger an
	183	+ * interrupt outside this iommufd context.
	184	+ */
	185	+ if (!iommufd_selftest_is_mock_dev(dev) &&
	186	+ !iommu_group_has_isolated_msi(igroup->group)) {
	187	+ if (!allow_unsafe_interrupts) {
	188	+ rc = -EPERM;
	189	+ goto out_group_put;
	190	+ }
	191	+
	192	+ dev_warn(
	193	+ dev,
	194	+ "MSI interrupts are not secure, they cannot be isolated by the platform. "
	195	+ "Check that platform features like interrupt remapping are enabled. "
	196	+ "Use the \"allow_unsafe_interrupts\" module parameter to override\n");
	197	+ }
62	198
63	199	rc = iommu_device_claim_dma_owner(dev, ictx);
64	200	if (rc)
@@ -77,8 +213,8 @@ struct iommufd_device iommufd_device_bind(struct iommufd_ctx ictx, · 12 unmodified lines
77	213	device_iommu_capable(dev, IOMMU_CAP_ENFORCE_CACHE_COHERENCY);
78	214	/* The calling driver is a user until iommufd_device_unbind() */
79	215	refcount_inc(&idev->obj.users);
80		− /* group refcount moves into iommufd_device */
81		− idev->group = group;
	216	+ /* igroup refcount moves into iommufd_device */
	217	+ idev->igroup = igroup;
82	218
83	219	/*
84	220	* If the caller fails after this success it must call
@@ -93,7 +229,7 @@ struct iommufd_device iommufd_device_bind(struct iommufd_ctx ictx, · 8 unmodified lines
93	229	out_release_owner:
94	230	iommu_device_release_dma_owner(dev);
95	231	out_group_put:
96		− iommu_group_put(group);
	232	+ iommufd_put_group(igroup);
97	233	return ERR_PTR(rc);
98	234	}
99	235	EXPORT_SYMBOL_NS_GPL(iommufd_device_bind, IOMMUFD);
@@ -118,7 +254,8 @@ bool iommufd_ctx_has_group(struct iommufd_ctx ictx, struct iommu_group group) · 18 unmodified lines
118	254	xa_lock(&ictx->objects);
119	255	xa_for_each(&ictx->objects, index, obj) {
120	256	if (obj->type == IOMMUFD_OBJ_DEVICE &&
121		− container_of(obj, struct iommufd_device, obj)->group == group) {
	257	+ container_of(obj, struct iommufd_device, obj)
	258	+ ->igroup->group == group) {
122	259	xa_unlock(&ictx->objects);
123	260	return true;
124	261	}
@@ -155,10 +292,10 @@ u32 iommufd_device_to_id(struct iommufd_device *idev) · 30 unmodified lines
155	292	}
156	293	EXPORT_SYMBOL_NS_GPL(iommufd_device_to_id, IOMMUFD);
157	294
158		−static int iommufd_device_setup_msi(struct iommufd_device *idev,
159		− struct iommufd_hw_pagetable *hwpt,
160		− phys_addr_t sw_msi_start)
	295	+static int iommufd_group_setup_msi(struct iommufd_group *igroup,
	296	+ struct iommufd_hw_pagetable *hwpt)
161	297	{
	298	+ phys_addr_t sw_msi_start = igroup->sw_msi_start;
162	299	int rc;
163	300
164	301	/*
@@ -185,128 +322,192 @@ static int iommufd_device_setup_msi(struct iommufd_device *idev, · 20 unmodified lines
185	322	*/
186	323	hwpt->msi_cookie = true;
187	324	}
188		−
189		− /*
190		− * For historical compat with VFIO the insecure interrupt path is
191		− * allowed if the module parameter is set. Insecure means that a MemWr
192		− * operation from the device (eg a simple DMA) cannot trigger an
193		− * interrupt outside this iommufd context.
194		− */
195		− if (!iommufd_selftest_is_mock_dev(idev->dev) &&
196		− !iommu_group_has_isolated_msi(idev->group)) {
197		− if (!allow_unsafe_interrupts)
198		− return -EPERM;
199		−
200		− dev_warn(
201		− idev->dev,
202		− "MSI interrupts are not secure, they cannot be isolated by the platform. "
203		− "Check that platform features like interrupt remapping are enabled. "
204		− "Use the \"allow_unsafe_interrupts\" module parameter to override\n");
205		− }
206	325	return 0;
207	326	}
208	327
209		−static bool iommufd_hw_pagetable_has_group(struct iommufd_hw_pagetable *hwpt,
210		− struct iommu_group *group)
211		−{
212		− struct iommufd_device *cur_dev;
213		−
214		− lockdep_assert_held(&hwpt->devices_lock);
215		−
216		− list_for_each_entry(cur_dev, &hwpt->devices, devices_item)
217		− if (cur_dev->group == group)
218		− return true;
219		− return false;
220		−}
221		−
222	328	int iommufd_hw_pagetable_attach(struct iommufd_hw_pagetable *hwpt,
223	329	struct iommufd_device *idev)
224	330	{
225		− phys_addr_t sw_msi_start = PHYS_ADDR_MAX;
226	331	int rc;
227	332
228		− lockdep_assert_held(&hwpt->devices_lock);
229		−
230		− if (WARN_ON(idev->hwpt))
231		− return -EINVAL;
	333	+ mutex_lock(&idev->igroup->lock);
232	334
233		− /*
234		− * Try to upgrade the domain we have, it is an iommu driver bug to
235		− * report IOMMU_CAP_ENFORCE_CACHE_COHERENCY but fail
236		− * enforce_cache_coherency when there are no devices attached to the
237		− * domain.
238		− */
239		− if (idev->enforce_cache_coherency && !hwpt->enforce_cache_coherency) {
240		− if (hwpt->domain->ops->enforce_cache_coherency)
241		− hwpt->enforce_cache_coherency =
242		− hwpt->domain->ops->enforce_cache_coherency(
243		− hwpt->domain);
244		− if (!hwpt->enforce_cache_coherency) {
245		− WARN_ON(list_empty(&hwpt->devices));
246		− return -EINVAL;
247		− }
	335	+ if (idev->igroup->hwpt != NULL && idev->igroup->hwpt != hwpt) {
	336	+ rc = -EINVAL;
	337	+ goto err_unlock;
248	338	}
249	339
250		− rc = iopt_table_enforce_group_resv_regions(&hwpt->ioas->iopt, idev->dev,
251		− idev->group, &sw_msi_start);
252		− if (rc)
253		− return rc;
	340	+ /* Try to upgrade the domain we have */
	341	+ if (idev->enforce_cache_coherency) {
	342	+ rc = iommufd_hw_pagetable_enforce_cc(hwpt);
	343	+ if (rc)
	344	+ goto err_unlock;
	345	+ }
254	346
255		− rc = iommufd_device_setup_msi(idev, hwpt, sw_msi_start);
	347	+ rc = iopt_table_enforce_dev_resv_regions(&hwpt->ioas->iopt, idev->dev,
	348	+ &idev->igroup->sw_msi_start);
256	349	if (rc)
257		− goto err_unresv;
	350	+ goto err_unlock;
258	351
259	352	/*
260		− * FIXME: Hack around missing a device-centric iommu api, only attach to
261		− * the group once for the first device that is in the group.
	353	+ * Only attach to the group once for the first device that is in the
	354	+ * group. All the other devices will follow this attachment. The user
	355	+ * should attach every device individually to the hwpt as the per-device
	356	+ * reserved regions are only updated during individual device
	357	+ * attachment.
262	358	*/
263		− if (!iommufd_hw_pagetable_has_group(hwpt, idev->group)) {
264		− rc = iommu_attach_group(hwpt->domain, idev->group);
	359	+ if (list_empty(&idev->igroup->device_list)) {
	360	+ rc = iommufd_group_setup_msi(idev->igroup, hwpt);
265	361	if (rc)
266	362	goto err_unresv;
	363	+
	364	+ rc = iommu_attach_group(hwpt->domain, idev->igroup->group);
	365	+ if (rc)
	366	+ goto err_unresv;
	367	+ idev->igroup->hwpt = hwpt;
267	368	}
	369	+ refcount_inc(&hwpt->obj.users);
	370	+ list_add_tail(&idev->group_item, &idev->igroup->device_list);
	371	+ mutex_unlock(&idev->igroup->lock);
268	372	return 0;
269	373	err_unresv:
270	374	iopt_remove_reserved_iova(&hwpt->ioas->iopt, idev->dev);
	375	+err_unlock:
	376	+ mutex_unlock(&idev->igroup->lock);
271	377	return rc;
272	378	}
273	379
274		−void iommufd_hw_pagetable_detach(struct iommufd_hw_pagetable *hwpt,
275		− struct iommufd_device *idev)
	380	+struct iommufd_hw_pagetable *
	381	+iommufd_hw_pagetable_detach(struct iommufd_device *idev)
276	382	{
277		− if (!iommufd_hw_pagetable_has_group(hwpt, idev->group))
278		− iommu_detach_group(hwpt->domain, idev->group);
	383	+ struct iommufd_hw_pagetable *hwpt = idev->igroup->hwpt;
	384	+
	385	+ mutex_lock(&idev->igroup->lock);
	386	+ list_del(&idev->group_item);
	387	+ if (list_empty(&idev->igroup->device_list)) {
	388	+ iommu_detach_group(hwpt->domain, idev->igroup->group);
	389	+ idev->igroup->hwpt = NULL;
	390	+ }
279	391	iopt_remove_reserved_iova(&hwpt->ioas->iopt, idev->dev);
	392	+ mutex_unlock(&idev->igroup->lock);
	393	+
	394	+ /* Caller must destroy hwpt */
	395	+ return hwpt;
280	396	}
281	397
282		−static int iommufd_device_do_attach(struct iommufd_device *idev,
283		− struct iommufd_hw_pagetable *hwpt)
	398	+static struct iommufd_hw_pagetable *
	399	+iommufd_device_do_attach(struct iommufd_device *idev,
	400	+ struct iommufd_hw_pagetable *hwpt)
284	401	{
285	402	int rc;
286	403
287		− mutex_lock(&hwpt->devices_lock);
288	404	rc = iommufd_hw_pagetable_attach(hwpt, idev);
289	405	if (rc)
290		− goto out_unlock;
	406	+ return ERR_PTR(rc);
	407	+ return NULL;
	408	+}
291	409
292		− idev->hwpt = hwpt;
293		− refcount_inc(&hwpt->obj.users);
294		− list_add(&idev->devices_item, &hwpt->devices);
295		−out_unlock:
296		− mutex_unlock(&hwpt->devices_lock);
297		− return rc;
	410	+static struct iommufd_hw_pagetable *
	411	+iommufd_device_do_replace(struct iommufd_device *idev,
	412	+ struct iommufd_hw_pagetable *hwpt)
	413	+{
	414	+ struct iommufd_group *igroup = idev->igroup;
	415	+ struct iommufd_hw_pagetable *old_hwpt;
	416	+ unsigned int num_devices = 0;
	417	+ struct iommufd_device *cur;
	418	+ int rc;
	419	+
	420	+ mutex_lock(&idev->igroup->lock);
	421	+
	422	+ if (igroup->hwpt == NULL) {
	423	+ rc = -EINVAL;
	424	+ goto err_unlock;
	425	+ }
	426	+
	427	+ if (hwpt == igroup->hwpt) {
	428	+ mutex_unlock(&idev->igroup->lock);
	429	+ return NULL;
	430	+ }
	431	+
	432	+ /* Try to upgrade the domain we have */
	433	+ list_for_each_entry(cur, &igroup->device_list, group_item) {
	434	+ num_devices++;
	435	+ if (cur->enforce_cache_coherency) {
	436	+ rc = iommufd_hw_pagetable_enforce_cc(hwpt);
	437	+ if (rc)
	438	+ goto err_unlock;
	439	+ }
	440	+ }
	441	+
	442	+ old_hwpt = igroup->hwpt;
	443	+ if (hwpt->ioas != old_hwpt->ioas) {
	444	+ list_for_each_entry(cur, &igroup->device_list, group_item) {
	445	+ rc = iopt_table_enforce_dev_resv_regions(
	446	+ &hwpt->ioas->iopt, cur->dev, NULL);
	447	+ if (rc)
	448	+ goto err_unresv;
	449	+ }
	450	+ }
	451	+
	452	+ rc = iommufd_group_setup_msi(idev->igroup, hwpt);
	453	+ if (rc)
	454	+ goto err_unresv;
	455	+
	456	+ rc = iommu_group_replace_domain(igroup->group, hwpt->domain);
	457	+ if (rc)
	458	+ goto err_unresv;
	459	+
	460	+ if (hwpt->ioas != old_hwpt->ioas) {
	461	+ list_for_each_entry(cur, &igroup->device_list, group_item)
	462	+ iopt_remove_reserved_iova(&old_hwpt->ioas->iopt,
	463	+ cur->dev);
	464	+ }
	465	+
	466	+ igroup->hwpt = hwpt;
	467	+
	468	+ /*
	469	+ * Move the refcounts held by the device_list to the new hwpt. Retain a
	470	+ * refcount for this thread as the caller will free it.
	471	+ */
	472	+ refcount_add(num_devices, &hwpt->obj.users);
	473	+ if (num_devices > 1)
	474	+ WARN_ON(refcount_sub_and_test(num_devices - 1,
	475	+ &old_hwpt->obj.users));
	476	+ mutex_unlock(&idev->igroup->lock);
	477	+
	478	+ /* Caller must destroy old_hwpt */
	479	+ return old_hwpt;
	480	+err_unresv:
	481	+ list_for_each_entry(cur, &igroup->device_list, group_item)
	482	+ iopt_remove_reserved_iova(&hwpt->ioas->iopt, cur->dev);
	483	+err_unlock:
	484	+ mutex_unlock(&idev->igroup->lock);
	485	+ return ERR_PTR(rc);
298	486	}
299	487
	488	+typedef struct iommufd_hw_pagetable (attach_fn)(
	489	+ struct iommufd_device idev, struct iommufd_hw_pagetable hwpt);
	490	+
300	491	/*
301	492	* When automatically managing the domains we search for a compatible domain in
302	493	* the iopt and if one is found use it, otherwise create a new domain.
303	494	* Automatic domain selection will never pick a manually created domain.
304	495	*/
305		−static int iommufd_device_auto_get_domain(struct iommufd_device *idev,
306		− struct iommufd_ioas *ioas)
	496	+static struct iommufd_hw_pagetable *
	497	+iommufd_device_auto_get_domain(struct iommufd_device *idev,
	498	+ struct iommufd_ioas ioas, u32 pt_id,
	499	+ attach_fn do_attach)
307	500	{
	501	+ /*
	502	+ * iommufd_hw_pagetable_attach() is called by
	503	+ * iommufd_hw_pagetable_alloc() in immediate attachment mode, same as
	504	+ * iommufd_device_do_attach(). So if we are in this mode then we prefer
	505	+ * to use the immediate_attach path as it supports drivers that can't
	506	+ * directly allocate a domain.
	507	+ */
	508	+ bool immediate_attach = do_attach == iommufd_device_do_attach;
	509	+ struct iommufd_hw_pagetable *destroy_hwpt;
308	510	struct iommufd_hw_pagetable *hwpt;
309		− int rc;
310	511
311	512	/*
312	513	* There is no differentiation when domains are allocated, so any domain
@@ -320,50 +521,58 @@ static int iommufd_device_auto_get_domain(struct iommufd_device *idev, · 7 unmodified lines
320	521
321	522	if (!iommufd_lock_obj(&hwpt->obj))
322	523	continue;
323		− rc = iommufd_device_do_attach(idev, hwpt);
	524	+ destroy_hwpt = (*do_attach)(idev, hwpt);
	525	+ if (IS_ERR(destroy_hwpt)) {
	526	+ iommufd_put_object(&hwpt->obj);
	527	+ /*
	528	+ * -EINVAL means the domain is incompatible with the
	529	+ * device. Other error codes should propagate to
	530	+ * userspace as failure. Success means the domain is
	531	+ * attached.
	532	+ */
	533	+ if (PTR_ERR(destroy_hwpt) == -EINVAL)
	534	+ continue;
	535	+ goto out_unlock;
	536	+ }
	537	+ *pt_id = hwpt->obj.id;
324	538	iommufd_put_object(&hwpt->obj);
325		−
326		− /*
327		− * -EINVAL means the domain is incompatible with the device.
328		− * Other error codes should propagate to userspace as failure.
329		− * Success means the domain is attached.
330		− */
331		− if (rc == -EINVAL)
332		− continue;
333	539	goto out_unlock;
334	540	}
335	541
336		− hwpt = iommufd_hw_pagetable_alloc(idev->ictx, ioas, idev, true);
	542	+ hwpt = iommufd_hw_pagetable_alloc(idev->ictx, ioas, idev,
	543	+ immediate_attach);
337	544	if (IS_ERR(hwpt)) {
338		− rc = PTR_ERR(hwpt);
	545	+ destroy_hwpt = ERR_CAST(hwpt);
339	546	goto out_unlock;
340	547	}
	548	+
	549	+ if (!immediate_attach) {
	550	+ destroy_hwpt = (*do_attach)(idev, hwpt);
	551	+ if (IS_ERR(destroy_hwpt))
	552	+ goto out_abort;
	553	+ } else {
	554	+ destroy_hwpt = NULL;
	555	+ }
	556	+
341	557	hwpt->auto_domain = true;
	558	+ *pt_id = hwpt->obj.id;
342	559
343		− mutex_unlock(&ioas->mutex);
344	560	iommufd_object_finalize(idev->ictx, &hwpt->obj);
345		− return 0;
	561	+ mutex_unlock(&ioas->mutex);
	562	+ return destroy_hwpt;
	563	+
	564	+out_abort:
	565	+ iommufd_object_abort_and_destroy(idev->ictx, &hwpt->obj);
346	566	out_unlock:
347	567	mutex_unlock(&ioas->mutex);
348		− return rc;
	568	+ return destroy_hwpt;
349	569	}
350	570
351		−/**
352		− * iommufd_device_attach - Connect a device from an iommu_domain
353		− * @idev: device to attach
354		− * @pt_id: Input a IOMMUFD_OBJ_IOAS, or IOMMUFD_OBJ_HW_PAGETABLE
355		− * Output the IOMMUFD_OBJ_HW_PAGETABLE ID
356		− *
357		− * This connects the device to an iommu_domain, either automatically or manually
358		− * selected. Once this completes the device could do DMA.
359		− *
360		− * The caller should return the resulting pt_id back to userspace.
361		− * This function is undone by calling iommufd_device_detach().
362		− */
363		−int iommufd_device_attach(struct iommufd_device idev, u32 pt_id)
	571	+static int iommufd_device_change_pt(struct iommufd_device idev, u32 pt_id,
	572	+ attach_fn do_attach)
364	573	{
	574	+ struct iommufd_hw_pagetable *destroy_hwpt;
365	575	struct iommufd_object *pt_obj;
366		− int rc;
367	576
368	577	pt_obj = iommufd_get_object(idev->ictx, *pt_id, IOMMUFD_OBJ_ANY);
369	578	if (IS_ERR(pt_obj))
@@ -374,8 +583,8 @@ int iommufd_device_attach(struct iommufd_device idev, u32 pt_id) · 4 unmodified lines
374	583	struct iommufd_hw_pagetable *hwpt =
375	584	container_of(pt_obj, struct iommufd_hw_pagetable, obj);
376	585
377		− rc = iommufd_device_do_attach(idev, hwpt);
378		− if (rc)
	586	+ destroy_hwpt = (*do_attach)(idev, hwpt);
	587	+ if (IS_ERR(destroy_hwpt))
379	588	goto out_put_pt_obj;
380	589	break;
381	590	}
@@ -383,26 +592,79 @@ int iommufd_device_attach(struct iommufd_device idev, u32 pt_id) · 1 unmodified line
383	592	struct iommufd_ioas *ioas =
384	593	container_of(pt_obj, struct iommufd_ioas, obj);
385	594
386		− rc = iommufd_device_auto_get_domain(idev, ioas);
387		− if (rc)
	595	+ destroy_hwpt = iommufd_device_auto_get_domain(idev, ioas, pt_id,
	596	+ do_attach);
	597	+ if (IS_ERR(destroy_hwpt))
388	598	goto out_put_pt_obj;
389	599	break;
390	600	}
391	601	default:
392		− rc = -EINVAL;
	602	+ destroy_hwpt = ERR_PTR(-EINVAL);
393	603	goto out_put_pt_obj;
394	604	}
	605	+ iommufd_put_object(pt_obj);
395	606
396		− refcount_inc(&idev->obj.users);
397		− *pt_id = idev->hwpt->obj.id;
398		− rc = 0;
	607	+ /* This destruction has to be after we unlock everything */
	608	+ if (destroy_hwpt)
	609	+ iommufd_hw_pagetable_put(idev->ictx, destroy_hwpt);
	610	+ return 0;
399	611
400	612	out_put_pt_obj:
401	613	iommufd_put_object(pt_obj);
402		− return rc;
	614	+ return PTR_ERR(destroy_hwpt);
	615	+}
	616	+
	617	+/**
	618	+ * iommufd_device_attach - Connect a device to an iommu_domain
	619	+ * @idev: device to attach
	620	+ * @pt_id: Input a IOMMUFD_OBJ_IOAS, or IOMMUFD_OBJ_HW_PAGETABLE
	621	+ * Output the IOMMUFD_OBJ_HW_PAGETABLE ID
	622	+ *
	623	+ * This connects the device to an iommu_domain, either automatically or manually
	624	+ * selected. Once this completes the device could do DMA.
	625	+ *
	626	+ * The caller should return the resulting pt_id back to userspace.
	627	+ * This function is undone by calling iommufd_device_detach().
	628	+ */
	629	+int iommufd_device_attach(struct iommufd_device idev, u32 pt_id)
	630	+{
	631	+ int rc;
	632	+
	633	+ rc = iommufd_device_change_pt(idev, pt_id, &iommufd_device_do_attach);
	634	+ if (rc)
	635	+ return rc;
	636	+
	637	+ /*
	638	+ * Pairs with iommufd_device_detach() - catches caller bugs attempting
	639	+ * to destroy a device with an attachment.
	640	+ */
	641	+ refcount_inc(&idev->obj.users);
	642	+ return 0;
403	643	}
404	644	EXPORT_SYMBOL_NS_GPL(iommufd_device_attach, IOMMUFD);
405	645
	646	+/**
	647	+ * iommufd_device_replace - Change the device's iommu_domain
	648	+ * @idev: device to change
	649	+ * @pt_id: Input a IOMMUFD_OBJ_IOAS, or IOMMUFD_OBJ_HW_PAGETABLE
	650	+ * Output the IOMMUFD_OBJ_HW_PAGETABLE ID
	651	+ *
	652	+ * This is the same as::
	653	+ *
	654	+ * iommufd_device_detach();
	655	+ * iommufd_device_attach();
	656	+ *
	657	+ * If it fails then no change is made to the attachment. The iommu driver may
	658	+ * implement this so there is no disruption in translation. This can only be
	659	+ * called if iommufd_device_attach() has already succeeded.
	660	+ */
	661	+int iommufd_device_replace(struct iommufd_device idev, u32 pt_id)
	662	+{
	663	+ return iommufd_device_change_pt(idev, pt_id,
	664	+ &iommufd_device_do_replace);
	665	+}
	666	+EXPORT_SYMBOL_NS_GPL(iommufd_device_replace, IOMMUFD);
	667	+
406	668	/**
407	669	* iommufd_device_detach - Disconnect a device to an iommu_domain
408	670	* @idev: device to detach
@@ -412,33 +674,87 @@ EXPORT_SYMBOL_NS_GPL(iommufd_device_attach, IOMMUFD); · 3 unmodified lines
412	674	*/
413	675	void iommufd_device_detach(struct iommufd_device *idev)
414	676	{
415		− struct iommufd_hw_pagetable *hwpt = idev->hwpt;
416		−
417		− mutex_lock(&hwpt->devices_lock);
418		− list_del(&idev->devices_item);
419		− idev->hwpt = NULL;
420		− iommufd_hw_pagetable_detach(hwpt, idev);
421		− mutex_unlock(&hwpt->devices_lock);
422		−
423		− if (hwpt->auto_domain)
424		− iommufd_object_deref_user(idev->ictx, &hwpt->obj);
425		− else
426		− refcount_dec(&hwpt->obj.users);
	677	+ struct iommufd_hw_pagetable *hwpt;
427	678
	679	+ hwpt = iommufd_hw_pagetable_detach(idev);
	680	+ iommufd_hw_pagetable_put(idev->ictx, hwpt);
428	681	refcount_dec(&idev->obj.users);
429	682	}
430	683	EXPORT_SYMBOL_NS_GPL(iommufd_device_detach, IOMMUFD);
431	684
	685	+/*
	686	+ * On success, it will refcount_inc() at a valid new_ioas and refcount_dec() at
	687	+ * a valid cur_ioas (access->ioas). A caller passing in a valid new_ioas should
	688	+ * call iommufd_put_object() if it does an iommufd_get_object() for a new_ioas.
	689	+ */
	690	+static int iommufd_access_change_ioas(struct iommufd_access *access,
	691	+ struct iommufd_ioas *new_ioas)
	692	+{
	693	+ u32 iopt_access_list_id = access->iopt_access_list_id;
	694	+ struct iommufd_ioas *cur_ioas = access->ioas;
	695	+ int rc;
	696	+
	697	+ lockdep_assert_held(&access->ioas_lock);
	698	+
	699	+ /* We are racing with a concurrent detach, bail */
	700	+ if (cur_ioas != access->ioas_unpin)
	701	+ return -EBUSY;
	702	+
	703	+ if (cur_ioas == new_ioas)
	704	+ return 0;
	705	+
	706	+ /*
	707	+ * Set ioas to NULL to block any further iommufd_access_pin_pages().
	708	+ * iommufd_access_unpin_pages() can continue using access->ioas_unpin.
	709	+ */
	710	+ access->ioas = NULL;
	711	+
	712	+ if (new_ioas) {
	713	+ rc = iopt_add_access(&new_ioas->iopt, access);
	714	+ if (rc) {
	715	+ access->ioas = cur_ioas;
	716	+ return rc;
	717	+ }
	718	+ refcount_inc(&new_ioas->obj.users);
	719	+ }
	720	+
	721	+ if (cur_ioas) {
	722	+ if (access->ops->unmap) {
	723	+ mutex_unlock(&access->ioas_lock);
	724	+ access->ops->unmap(access->data, 0, ULONG_MAX);
	725	+ mutex_lock(&access->ioas_lock);
	726	+ }
	727	+ iopt_remove_access(&cur_ioas->iopt, access, iopt_access_list_id);
	728	+ refcount_dec(&cur_ioas->obj.users);
	729	+ }
	730	+
	731	+ access->ioas = new_ioas;
	732	+ access->ioas_unpin = new_ioas;
	733	+
	734	+ return 0;
	735	+}
	736	+
	737	+static int iommufd_access_change_ioas_id(struct iommufd_access *access, u32 id)
	738	+{
	739	+ struct iommufd_ioas *ioas = iommufd_get_ioas(access->ictx, id);
	740	+ int rc;
	741	+
	742	+ if (IS_ERR(ioas))
	743	+ return PTR_ERR(ioas);
	744	+ rc = iommufd_access_change_ioas(access, ioas);
	745	+ iommufd_put_object(&ioas->obj);
	746	+ return rc;
	747	+}
	748	+
432	749	void iommufd_access_destroy_object(struct iommufd_object *obj)
433	750	{
434	751	struct iommufd_access *access =
435	752	container_of(obj, struct iommufd_access, obj);
436	753
437		− if (access->ioas) {
438		− iopt_remove_access(&access->ioas->iopt, access);
439		− refcount_dec(&access->ioas->obj.users);
440		− access->ioas = NULL;
441		− }
	754	+ mutex_lock(&access->ioas_lock);
	755	+ if (access->ioas)
	756	+ WARN_ON(iommufd_access_change_ioas(access, NULL));
	757	+ mutex_unlock(&access->ioas_lock);
442	758	iommufd_ctx_put(access->ictx);
443	759	}
444	760
@@ -502,61 +818,46 @@ EXPORT_SYMBOL_NS_GPL(iommufd_access_destroy, IOMMUFD); · 57 unmodified lines
502	818
503	819	void iommufd_access_detach(struct iommufd_access *access)
504	820	{
505		− struct iommufd_ioas *cur_ioas = access->ioas;
506		−
507	821	mutex_lock(&access->ioas_lock);
508		− if (WARN_ON(!access->ioas))
509		− goto out;
510		− /*
511		− * Set ioas to NULL to block any further iommufd_access_pin_pages().
512		− * iommufd_access_unpin_pages() can continue using access->ioas_unpin.
513		− */
514		− access->ioas = NULL;
515		−
516		− if (access->ops->unmap) {
	822	+ if (WARN_ON(!access->ioas)) {
517	823	mutex_unlock(&access->ioas_lock);
518		− access->ops->unmap(access->data, 0, ULONG_MAX);
519		− mutex_lock(&access->ioas_lock);
	824	+ return;
520	825	}
521		− iopt_remove_access(&cur_ioas->iopt, access);
522		− refcount_dec(&cur_ioas->obj.users);
523		−out:
524		− access->ioas_unpin = NULL;
	826	+ WARN_ON(iommufd_access_change_ioas(access, NULL));
525	827	mutex_unlock(&access->ioas_lock);
526	828	}
527	829	EXPORT_SYMBOL_NS_GPL(iommufd_access_detach, IOMMUFD);
528	830
529	831	int iommufd_access_attach(struct iommufd_access *access, u32 ioas_id)
530	832	{
531		− struct iommufd_ioas *new_ioas;
532		− int rc = 0;
	833	+ int rc;
533	834
534	835	mutex_lock(&access->ioas_lock);
535		− if (WARN_ON(access->ioas \|\| access->ioas_unpin)) {
	836	+ if (WARN_ON(access->ioas)) {
536	837	mutex_unlock(&access->ioas_lock);
537	838	return -EINVAL;
538	839	}
539	840
540		− new_ioas = iommufd_get_ioas(access->ictx, ioas_id);
541		− if (IS_ERR(new_ioas)) {
542		− mutex_unlock(&access->ioas_lock);
543		− return PTR_ERR(new_ioas);
544		− }
	841	+ rc = iommufd_access_change_ioas_id(access, ioas_id);
	842	+ mutex_unlock(&access->ioas_lock);
	843	+ return rc;
	844	+}
	845	+EXPORT_SYMBOL_NS_GPL(iommufd_access_attach, IOMMUFD);
	846	+
	847	+int iommufd_access_replace(struct iommufd_access *access, u32 ioas_id)
	848	+{
	849	+ int rc;
545	850
546		− rc = iopt_add_access(&new_ioas->iopt, access);
547		− if (rc) {
	851	+ mutex_lock(&access->ioas_lock);
	852	+ if (!access->ioas) {
548	853	mutex_unlock(&access->ioas_lock);
549		− iommufd_put_object(&new_ioas->obj);
550		− return rc;
	854	+ return -ENOENT;
551	855	}
552		− iommufd_ref_to_users(&new_ioas->obj);
553		−
554		− access->ioas = new_ioas;
555		− access->ioas_unpin = new_ioas;
	856	+ rc = iommufd_access_change_ioas_id(access, ioas_id);
556	857	mutex_unlock(&access->ioas_lock);
557		− return 0;
	858	+ return rc;
558	859	}
559		−EXPORT_SYMBOL_NS_GPL(iommufd_access_attach, IOMMUFD);
	860	+EXPORT_SYMBOL_NS_GPL(iommufd_access_replace, IOMMUFD);
560	861
561	862	/**
562	863	* iommufd_access_notify_unmap - Notify users of an iopt to stop using it
@@ -819,3 +1120,75 @@ err_out: · 256 unmodified lines
819	1120	return rc;
820	1121	}
821	1122	EXPORT_SYMBOL_NS_GPL(iommufd_access_rw, IOMMUFD);
	1123	+
	1124	+int iommufd_get_hw_info(struct iommufd_ucmd *ucmd)
	1125	+{
	1126	+ struct iommu_hw_info *cmd = ucmd->cmd;
	1127	+ void __user *user_ptr = u64_to_user_ptr(cmd->data_uptr);
	1128	+ const struct iommu_ops *ops;
	1129	+ struct iommufd_device *idev;
	1130	+ unsigned int data_len;
	1131	+ unsigned int copy_len;
	1132	+ void *data;
	1133	+ int rc;
	1134	+
	1135	+ if (cmd->flags \|\| cmd->__reserved)
	1136	+ return -EOPNOTSUPP;
	1137	+
	1138	+ idev = iommufd_get_device(ucmd, cmd->dev_id);
	1139	+ if (IS_ERR(idev))
	1140	+ return PTR_ERR(idev);
	1141	+
	1142	+ ops = dev_iommu_ops(idev->dev);
	1143	+ if (ops->hw_info) {
	1144	+ data = ops->hw_info(idev->dev, &data_len, &cmd->out_data_type);
	1145	+ if (IS_ERR(data)) {
	1146	+ rc = PTR_ERR(data);
	1147	+ goto out_put;
	1148	+ }
	1149	+
	1150	+ /*
	1151	+ * drivers that have hw_info callback should have a unique
	1152	+ * iommu_hw_info_type.
	1153	+ */
	1154	+ if (WARN_ON_ONCE(cmd->out_data_type ==
	1155	+ IOMMU_HW_INFO_TYPE_NONE)) {
	1156	+ rc = -ENODEV;
	1157	+ goto out_free;
	1158	+ }
	1159	+ } else {
	1160	+ cmd->out_data_type = IOMMU_HW_INFO_TYPE_NONE;
	1161	+ data_len = 0;
	1162	+ data = NULL;
	1163	+ }
	1164	+
	1165	+ copy_len = min(cmd->data_len, data_len);
	1166	+ if (copy_to_user(user_ptr, data, copy_len)) {
	1167	+ rc = -EFAULT;
	1168	+ goto out_free;
	1169	+ }
	1170	+
	1171	+ /*
	1172	+ * Zero the trailing bytes if the user buffer is bigger than the
	1173	+ * data size kernel actually has.
	1174	+ */
	1175	+ if (copy_len < cmd->data_len) {
	1176	+ if (clear_user(user_ptr + copy_len, cmd->data_len - copy_len)) {
	1177	+ rc = -EFAULT;
	1178	+ goto out_free;
	1179	+ }
	1180	+ }
	1181	+
	1182	+ /*
	1183	+ * We return the length the kernel supports so userspace may know what
	1184	+ * the kernel capability is. It could be larger than the input buffer.
	1185	+ */
	1186	+ cmd->data_len = data_len;
	1187	+
	1188	+ rc = iommufd_ucmd_respond(ucmd, sizeof(*cmd));
	1189	+out_free:
	1190	+ kfree(data);
	1191	+out_put:
	1192	+ iommufd_put_object(&idev->obj);
	1193	+ return rc;
	1194	+}

drivers/iommu/iommufd/hw_pagetable.c+93−19

@@ -3,6 +3,7 @@ · 2 unmodified lines
3	3	* Copyright (c) 2021-2022, NVIDIA CORPORATION & AFFILIATES
4	4	*/
5	5	#include <linux/iommu.h>
	6	+#include <uapi/linux/iommufd.h>
6	7
7	8	#include "iommufd_private.h"
8	9
@@ -11,8 +12,6 @@ void iommufd_hw_pagetable_destroy(struct iommufd_object *obj) · 2 unmodified lines
11	12	struct iommufd_hw_pagetable *hwpt =
12	13	container_of(obj, struct iommufd_hw_pagetable, obj);
13	14
14		− WARN_ON(!list_empty(&hwpt->devices));
15		−
16	15	if (!list_empty(&hwpt->hwpt_item)) {
17	16	mutex_lock(&hwpt->ioas->mutex);
18	17	list_del(&hwpt->hwpt_item);
@@ -25,7 +24,35 @@ void iommufd_hw_pagetable_destroy(struct iommufd_object *obj) · 6 unmodified lines
25	24	iommu_domain_free(hwpt->domain);
26	25
27	26	refcount_dec(&hwpt->ioas->obj.users);
28		− mutex_destroy(&hwpt->devices_lock);
	27	+}
	28	+
	29	+void iommufd_hw_pagetable_abort(struct iommufd_object *obj)
	30	+{
	31	+ struct iommufd_hw_pagetable *hwpt =
	32	+ container_of(obj, struct iommufd_hw_pagetable, obj);
	33	+
	34	+ /* The ioas->mutex must be held until finalize is called. */
	35	+ lockdep_assert_held(&hwpt->ioas->mutex);
	36	+
	37	+ if (!list_empty(&hwpt->hwpt_item)) {
	38	+ list_del_init(&hwpt->hwpt_item);
	39	+ iopt_table_remove_domain(&hwpt->ioas->iopt, hwpt->domain);
	40	+ }
	41	+ iommufd_hw_pagetable_destroy(obj);
	42	+}
	43	+
	44	+int iommufd_hw_pagetable_enforce_cc(struct iommufd_hw_pagetable *hwpt)
	45	+{
	46	+ if (hwpt->enforce_cache_coherency)
	47	+ return 0;
	48	+
	49	+ if (hwpt->domain->ops->enforce_cache_coherency)
	50	+ hwpt->enforce_cache_coherency =
	51	+ hwpt->domain->ops->enforce_cache_coherency(
	52	+ hwpt->domain);
	53	+ if (!hwpt->enforce_cache_coherency)
	54	+ return -EINVAL;
	55	+ return 0;
29	56	}
30	57
31	58	/**
@@ -38,6 +65,10 @@ void iommufd_hw_pagetable_destroy(struct iommufd_object *obj) · 6 unmodified lines
38	65	* Allocate a new iommu_domain and return it as a hw_pagetable. The HWPT
39	66	* will be linked to the given ioas and upon return the underlying iommu_domain
40	67	* is fully popoulated.
	68	+ *
	69	+ * The caller must hold the ioas->mutex until after
	70	+ * iommufd_object_abort_and_destroy() or iommufd_object_finalize() is called on
	71	+ * the returned hwpt.
41	72	*/
42	73	struct iommufd_hw_pagetable *
43	74	iommufd_hw_pagetable_alloc(struct iommufd_ctx ictx, struct iommufd_ioas ioas,
@@ -52,9 +83,7 @@ iommufd_hw_pagetable_alloc(struct iommufd_ctx ictx, struct iommufd_ioas ioas, · 8 unmodified lines
52	83	if (IS_ERR(hwpt))
53	84	return hwpt;
54	85
55		− INIT_LIST_HEAD(&hwpt->devices);
56	86	INIT_LIST_HEAD(&hwpt->hwpt_item);
57		− mutex_init(&hwpt->devices_lock);
58	87	/* Pairs with iommufd_hw_pagetable_destroy() */
59	88	refcount_inc(&ioas->obj.users);
60	89	hwpt->ioas = ioas;
@@ -65,7 +94,18 @@ iommufd_hw_pagetable_alloc(struct iommufd_ctx ictx, struct iommufd_ioas ioas, · 4 unmodified lines
65	94	goto out_abort;
66	95	}
67	96
68		− mutex_lock(&hwpt->devices_lock);
	97	+ /*
	98	+ * Set the coherency mode before we do iopt_table_add_domain() as some
	99	+ * iommus have a per-PTE bit that controls it and need to decide before
	100	+ * doing any maps. It is an iommu driver bug to report
	101	+ * IOMMU_CAP_ENFORCE_CACHE_COHERENCY but fail enforce_cache_coherency on
	102	+ * a new domain.
	103	+ */
	104	+ if (idev->enforce_cache_coherency) {
	105	+ rc = iommufd_hw_pagetable_enforce_cc(hwpt);
	106	+ if (WARN_ON(rc))
	107	+ goto out_abort;
	108	+ }
69	109
70	110	/*
71	111	* immediate_attach exists only to accommodate iommu drivers that cannot
@@ -76,30 +116,64 @@ iommufd_hw_pagetable_alloc(struct iommufd_ctx ictx, struct iommufd_ioas ioas, · 4 unmodified lines
76	116	if (immediate_attach) {
77	117	rc = iommufd_hw_pagetable_attach(hwpt, idev);
78	118	if (rc)
79		− goto out_unlock;
	119	+ goto out_abort;
80	120	}
81	121
82	122	rc = iopt_table_add_domain(&hwpt->ioas->iopt, hwpt->domain);
83	123	if (rc)
84	124	goto out_detach;
85	125	list_add_tail(&hwpt->hwpt_item, &hwpt->ioas->hwpt_list);
86		−
87		− if (immediate_attach) {
88		− /* See iommufd_device_do_attach() */
89		− refcount_inc(&hwpt->obj.users);
90		− idev->hwpt = hwpt;
91		− list_add(&idev->devices_item, &hwpt->devices);
92		− }
93		−
94		− mutex_unlock(&hwpt->devices_lock);
95	126	return hwpt;
96	127
97	128	out_detach:
98	129	if (immediate_attach)
99		− iommufd_hw_pagetable_detach(hwpt, idev);
100		−out_unlock:
101		− mutex_unlock(&hwpt->devices_lock);
	130	+ iommufd_hw_pagetable_detach(idev);
102	131	out_abort:
103	132	iommufd_object_abort_and_destroy(ictx, &hwpt->obj);
104	133	return ERR_PTR(rc);
105	134	}
	135	+
	136	+int iommufd_hwpt_alloc(struct iommufd_ucmd *ucmd)
	137	+{
	138	+ struct iommu_hwpt_alloc *cmd = ucmd->cmd;
	139	+ struct iommufd_hw_pagetable *hwpt;
	140	+ struct iommufd_device *idev;
	141	+ struct iommufd_ioas *ioas;
	142	+ int rc;
	143	+
	144	+ if (cmd->flags \|\| cmd->__reserved)
	145	+ return -EOPNOTSUPP;
	146	+
	147	+ idev = iommufd_get_device(ucmd, cmd->dev_id);
	148	+ if (IS_ERR(idev))
	149	+ return PTR_ERR(idev);
	150	+
	151	+ ioas = iommufd_get_ioas(ucmd->ictx, cmd->pt_id);
	152	+ if (IS_ERR(ioas)) {
	153	+ rc = PTR_ERR(ioas);
	154	+ goto out_put_idev;
	155	+ }
	156	+
	157	+ mutex_lock(&ioas->mutex);
	158	+ hwpt = iommufd_hw_pagetable_alloc(ucmd->ictx, ioas, idev, false);
	159	+ if (IS_ERR(hwpt)) {
	160	+ rc = PTR_ERR(hwpt);
	161	+ goto out_unlock;
	162	+ }
	163	+
	164	+ cmd->out_hwpt_id = hwpt->obj.id;
	165	+ rc = iommufd_ucmd_respond(ucmd, sizeof(*cmd));
	166	+ if (rc)
	167	+ goto out_hwpt;
	168	+ iommufd_object_finalize(ucmd->ictx, &hwpt->obj);
	169	+ goto out_unlock;
	170	+
	171	+out_hwpt:
	172	+ iommufd_object_abort_and_destroy(ucmd->ictx, &hwpt->obj);
	173	+out_unlock:
	174	+ mutex_unlock(&ioas->mutex);
	175	+ iommufd_put_object(&ioas->obj);
	176	+out_put_idev:
	177	+ iommufd_put_object(&idev->obj);
	178	+ return rc;
	179	+}

drivers/iommu/iommufd/io_pagetable.c+17−19

@@ -1158,36 +1158,36 @@ out_unlock: · 1157 unmodified lines
1158	1158	}
1159	1159
1160	1160	void iopt_remove_access(struct io_pagetable *iopt,
1161		− struct iommufd_access *access)
	1161	+ struct iommufd_access *access,
	1162	+ u32 iopt_access_list_id)
1162	1163	{
1163	1164	down_write(&iopt->domains_rwsem);
1164	1165	down_write(&iopt->iova_rwsem);
1165		− WARN_ON(xa_erase(&iopt->access_list, access->iopt_access_list_id) !=
1166		− access);
	1166	+ WARN_ON(xa_erase(&iopt->access_list, iopt_access_list_id) != access);
1167	1167	WARN_ON(iopt_calculate_iova_alignment(iopt));
1168	1168	up_write(&iopt->iova_rwsem);
1169	1169	up_write(&iopt->domains_rwsem);
1170	1170	}
1171	1171
1172		−/* Narrow the valid_iova_itree to include reserved ranges from a group. */
1173		−int iopt_table_enforce_group_resv_regions(struct io_pagetable *iopt,
1174		− struct device *device,
1175		− struct iommu_group *group,
1176		− phys_addr_t *sw_msi_start)
	1172	+/* Narrow the valid_iova_itree to include reserved ranges from a device. */
	1173	+int iopt_table_enforce_dev_resv_regions(struct io_pagetable *iopt,
	1174	+ struct device *dev,
	1175	+ phys_addr_t *sw_msi_start)
1177	1176	{
1178	1177	struct iommu_resv_region *resv;
1179		− struct iommu_resv_region *tmp;
1180		− LIST_HEAD(group_resv_regions);
	1178	+ LIST_HEAD(resv_regions);
1181	1179	unsigned int num_hw_msi = 0;
1182	1180	unsigned int num_sw_msi = 0;
1183	1181	int rc;
1184	1182
	1183	+ if (iommufd_should_fail())
	1184	+ return -EINVAL;
	1185	+
1185	1186	down_write(&iopt->iova_rwsem);
1186		− rc = iommu_get_group_resv_regions(group, &group_resv_regions);
1187		− if (rc)
1188		− goto out_unlock;
	1187	+ /* FIXME: drivers allocate memory but there is no failure propogated */
	1188	+ iommu_get_resv_regions(dev, &resv_regions);
1189	1189
1190		− list_for_each_entry(resv, &group_resv_regions, list) {
	1190	+ list_for_each_entry(resv, &resv_regions, list) {
1191	1191	if (resv->type == IOMMU_RESV_DIRECT_RELAXABLE)
1192	1192	continue;
1193	1193
@@ -1199,7 +1199,7 @@ int iopt_table_enforce_group_resv_regions(struct io_pagetable *iopt, · 5 unmodified lines
1199	1199	}
1200	1200
1201	1201	rc = iopt_reserve_iova(iopt, resv->start,
1202		− resv->length - 1 + resv->start, device);
	1202	+ resv->length - 1 + resv->start, dev);
1203	1203	if (rc)
1204	1204	goto out_reserved;
1205	1205	}
@@ -1214,11 +1214,9 @@ int iopt_table_enforce_group_resv_regions(struct io_pagetable *iopt, · 8 unmodified lines
1214	1214	goto out_free_resv;
1215	1215
1216	1216	out_reserved:
1217		− __iopt_remove_reserved_iova(iopt, device);
	1217	+ __iopt_remove_reserved_iova(iopt, dev);
1218	1218	out_free_resv:
1219		− list_for_each_entry_safe(resv, tmp, &group_resv_regions, list)
1220		− kfree(resv);
1221		−out_unlock:
	1219	+ iommu_put_resv_regions(dev, &resv_regions);
1222	1220	up_write(&iopt->iova_rwsem);
1223	1221	return rc;
1224	1222	}

drivers/iommu/iommufd/iommufd_private.h+46−38

@@ -17,6 +17,7 @@ struct iommufd_device; · 16 unmodified lines
17	17	struct iommufd_ctx {
18	18	struct file *file;
19	19	struct xarray objects;
	20	+ struct xarray groups;
20	21
21	22	u8 account_mode;
22	23	/* Compatibility with VFIO no iommu */
@@ -75,10 +76,9 @@ int iopt_table_add_domain(struct io_pagetable *iopt, · 52 unmodified lines
75	76	struct iommu_domain *domain);
76	77	void iopt_table_remove_domain(struct io_pagetable *iopt,
77	78	struct iommu_domain *domain);
78		−int iopt_table_enforce_group_resv_regions(struct io_pagetable *iopt,
79		− struct device *device,
80		− struct iommu_group *group,
81		− phys_addr_t *sw_msi_start);
	79	+int iopt_table_enforce_dev_resv_regions(struct io_pagetable *iopt,
	80	+ struct device *dev,
	81	+ phys_addr_t *sw_msi_start);
82	82	int iopt_set_allow_iova(struct io_pagetable *iopt,
83	83	struct rb_root_cached *allowed_iova);
84	84	int iopt_reserve_iova(struct io_pagetable *iopt, unsigned long start,
@@ -119,6 +119,7 @@ enum iommufd_object_type { · 34 unmodified lines
119	119	#ifdef CONFIG_IOMMUFD_TEST
120	120	IOMMUFD_OBJ_SELFTEST,
121	121	#endif
	122	+ IOMMUFD_OBJ_MAX,
122	123	};
123	124
124	125	/* Base struct for all objects with a userspace ID handle. */
@@ -148,29 +149,6 @@ static inline void iommufd_put_object(struct iommufd_object *obj) · 23 unmodified lines
148	149	up_read(&obj->destroy_rwsem);
149	150	}
150	151
151		−/**
152		− * iommufd_ref_to_users() - Switch from destroy_rwsem to users refcount
153		− * protection
154		− * @obj - Object to release
155		− *
156		− * Objects have two refcount protections (destroy_rwsem and the refcount_t
157		− * users). Holding either of these will prevent the object from being destroyed.
158		− *
159		− * Depending on the use case, one protection or the other is appropriate. In
160		− * most cases references are being protected by the destroy_rwsem. This allows
161		− * orderly destruction of the object because iommufd_object_destroy_user() will
162		− * wait for it to become unlocked. However, as a rwsem, it cannot be held across
163		− * a system call return. So cases that have longer term needs must switch
164		− * to the weaker users refcount_t.
165		− *
166		− * With users protection iommufd_object_destroy_user() will return false,
167		− * refusing to destroy the object, causing -EBUSY to userspace.
168		− */
169		−static inline void iommufd_ref_to_users(struct iommufd_object *obj)
170		−{
171		− up_read(&obj->destroy_rwsem);
172		− /* iommufd_lock_obj() obtains users as well */
173		−}
174	152	void iommufd_object_abort(struct iommufd_ctx ictx, struct iommufd_object obj);
175	153	void iommufd_object_abort_and_destroy(struct iommufd_ctx *ictx,
176	154	struct iommufd_object *obj);
@@ -260,18 +238,39 @@ struct iommufd_hw_pagetable { · 83 unmodified lines
260	238	bool msi_cookie : 1;
261	239	/* Head at iommufd_ioas::hwpt_list */
262	240	struct list_head hwpt_item;
263		− struct mutex devices_lock;
264		− struct list_head devices;
265	241	};
266	242
267	243	struct iommufd_hw_pagetable *
268	244	iommufd_hw_pagetable_alloc(struct iommufd_ctx ictx, struct iommufd_ioas ioas,
269	245	struct iommufd_device *idev, bool immediate_attach);
	246	+int iommufd_hw_pagetable_enforce_cc(struct iommufd_hw_pagetable *hwpt);
270	247	int iommufd_hw_pagetable_attach(struct iommufd_hw_pagetable *hwpt,
271	248	struct iommufd_device *idev);
272		−void iommufd_hw_pagetable_detach(struct iommufd_hw_pagetable *hwpt,
273		− struct iommufd_device *idev);
	249	+struct iommufd_hw_pagetable *
	250	+iommufd_hw_pagetable_detach(struct iommufd_device *idev);
274	251	void iommufd_hw_pagetable_destroy(struct iommufd_object *obj);
	252	+void iommufd_hw_pagetable_abort(struct iommufd_object *obj);
	253	+int iommufd_hwpt_alloc(struct iommufd_ucmd *ucmd);
	254	+
	255	+static inline void iommufd_hw_pagetable_put(struct iommufd_ctx *ictx,
	256	+ struct iommufd_hw_pagetable *hwpt)
	257	+{
	258	+ lockdep_assert_not_held(&hwpt->ioas->mutex);
	259	+ if (hwpt->auto_domain)
	260	+ iommufd_object_deref_user(ictx, &hwpt->obj);
	261	+ else
	262	+ refcount_dec(&hwpt->obj.users);
	263	+}
	264	+
	265	+struct iommufd_group {
	266	+ struct kref ref;
	267	+ struct mutex lock;
	268	+ struct iommufd_ctx *ictx;
	269	+ struct iommu_group *group;
	270	+ struct iommufd_hw_pagetable *hwpt;
	271	+ struct list_head device_list;
	272	+ phys_addr_t sw_msi_start;
	273	+};
275	274
276	275	/*
277	276	* A iommufd_device object represents the binding relationship between a
@@ -281,16 +280,23 @@ void iommufd_hw_pagetable_destroy(struct iommufd_object *obj); · 3 unmodified lines
281	280	struct iommufd_device {
282	281	struct iommufd_object obj;
283	282	struct iommufd_ctx *ictx;
284		− struct iommufd_hw_pagetable *hwpt;
285		− /* Head at iommufd_hw_pagetable::devices */
286		− struct list_head devices_item;
	283	+ struct iommufd_group *igroup;
	284	+ struct list_head group_item;
287	285	/* always the physical device */
288	286	struct device *dev;
289		− struct iommu_group *group;
290	287	bool enforce_cache_coherency;
291	288	};
292	289
	290	+static inline struct iommufd_device *
	291	+iommufd_get_device(struct iommufd_ucmd *ucmd, u32 id)
	292	+{
	293	+ return container_of(iommufd_get_object(ucmd->ictx, id,
	294	+ IOMMUFD_OBJ_DEVICE),
	295	+ struct iommufd_device, obj);
	296	+}
	297	+
293	298	void iommufd_device_destroy(struct iommufd_object *obj);
	299	+int iommufd_get_hw_info(struct iommufd_ucmd *ucmd);
294	300
295	301	struct iommufd_access {
296	302	struct iommufd_object obj;
@@ -306,7 +312,8 @@ struct iommufd_access { · 9 unmodified lines
306	312
307	313	int iopt_add_access(struct io_pagetable iopt, struct iommufd_access access);
308	314	void iopt_remove_access(struct io_pagetable *iopt,
309		− struct iommufd_access *access);
	315	+ struct iommufd_access *access,
	316	+ u32 iopt_access_list_id);
310	317	void iommufd_access_destroy_object(struct iommufd_object *obj);
311	318
312	319	#ifdef CONFIG_IOMMUFD_TEST
@@ -316,7 +323,7 @@ extern size_t iommufd_test_memory_limit; · 3 unmodified lines
316	323	void iommufd_test_syz_conv_iova_id(struct iommufd_ucmd *ucmd,
317	324	unsigned int ioas_id, u64 iova, u32 flags);
318	325	bool iommufd_should_fail(void);
319		−void __init iommufd_test_init(void);
	326	+int __init iommufd_test_init(void);
320	327	void iommufd_test_exit(void);
321	328	bool iommufd_selftest_is_mock_dev(struct device *dev);
322	329	#else
@@ -329,8 +336,9 @@ static inline bool iommufd_should_fail(void) · 6 unmodified lines
329	336	{
330	337	return false;
331	338	}
332		−static inline void __init iommufd_test_init(void)
	339	+static inline int __init iommufd_test_init(void)
333	340	{
	341	+ return 0;
334	342	}
335	343	static inline void iommufd_test_exit(void)
336	344	{

drivers/iommu/iommufd/iommufd_test.h+19−0

@@ -17,6 +17,8 @@ enum { · 16 unmodified lines
17	17	IOMMU_TEST_OP_ACCESS_PAGES,
18	18	IOMMU_TEST_OP_ACCESS_RW,
19	19	IOMMU_TEST_OP_SET_TEMP_MEMORY_LIMIT,
	20	+ IOMMU_TEST_OP_MOCK_DOMAIN_REPLACE,
	21	+ IOMMU_TEST_OP_ACCESS_REPLACE_IOAS,
20	22	};
21	23
22	24	enum {
@@ -51,7 +53,12 @@ struct iommu_test_cmd { · 28 unmodified lines
51	53	struct {
52	54	__u32 out_stdev_id;
53	55	__u32 out_hwpt_id;
	56	+ /* out_idev_id is the standard iommufd_bind object */
	57	+ __u32 out_idev_id;
54	58	} mock_domain;
	59	+ struct {
	60	+ __u32 pt_id;
	61	+ } mock_domain_replace;
55	62	struct {
56	63	__aligned_u64 iova;
57	64	__aligned_u64 length;
@@ -85,9 +92,21 @@ struct iommu_test_cmd { · 27 unmodified lines
85	92	struct {
86	93	__u32 limit;
87	94	} memory_limit;
	95	+ struct {
	96	+ __u32 ioas_id;
	97	+ } access_replace_ioas;
88	98	};
89	99	__u32 last;
90	100	};
91	101	#define IOMMU_TEST_CMD _IO(IOMMUFD_TYPE, IOMMUFD_CMD_BASE + 32)
92	102
	103	+/* Mock structs for IOMMU_DEVICE_GET_HW_INFO ioctl */
	104	+#define IOMMU_HW_INFO_TYPE_SELFTEST 0xfeedbeef
	105	+#define IOMMU_HW_INFO_SELFTEST_REGVAL 0xdeadbeef
	106	+
	107	+struct iommu_test_hw_info {
	108	+ __u32 flags;
	109	+ __u32 test_reg;
	110	+};
	111	+
93	112	#endif

drivers/iommu/iommufd/main.c+32−3

@@ -24,6 +24,7 @@ · 23 unmodified lines
24	24
25	25	struct iommufd_object_ops {
26	26	void (destroy)(struct iommufd_object obj);
	27	+ void (abort)(struct iommufd_object obj);
27	28	};
28	29	static const struct iommufd_object_ops iommufd_object_ops[];
29	30	static struct miscdevice vfio_misc_dev;
@@ -32,6 +33,7 @@ struct iommufd_object _iommufd_object_alloc(struct iommufd_ctx ictx, · 2 unmodified lines
32	33	size_t size,
33	34	enum iommufd_object_type type)
34	35	{
	36	+ static struct lock_class_key obj_keys[IOMMUFD_OBJ_MAX];
35	37	struct iommufd_object *obj;
36	38	int rc;
37	39
@@ -39,7 +41,15 @@ struct iommufd_object _iommufd_object_alloc(struct iommufd_ctx ictx, · 1 unmodified line
39	41	if (!obj)
40	42	return ERR_PTR(-ENOMEM);
41	43	obj->type = type;
42		− init_rwsem(&obj->destroy_rwsem);
	44	+ /*
	45	+ * In most cases the destroy_rwsem is obtained with try so it doesn't
	46	+ * interact with lockdep, however on destroy we have to sleep. This
	47	+ * means if we have to destroy an object while holding a get on another
	48	+ * object it triggers lockdep. Using one locking class per object type
	49	+ * is a simple and reasonable way to avoid this.
	50	+ */
	51	+ __init_rwsem(&obj->destroy_rwsem, "iommufd_object::destroy_rwsem",
	52	+ &obj_keys[type]);
43	53	refcount_set(&obj->users, 1);
44	54
45	55	/*
@@ -95,7 +105,10 @@ void iommufd_object_abort(struct iommufd_ctx ictx, struct iommufd_object obj) · 49 unmodified lines
95	105	void iommufd_object_abort_and_destroy(struct iommufd_ctx *ictx,
96	106	struct iommufd_object *obj)
97	107	{
98		− iommufd_object_ops[obj->type].destroy(obj);
	108	+ if (iommufd_object_ops[obj->type].abort)
	109	+ iommufd_object_ops[obj->type].abort(obj);
	110	+ else
	111	+ iommufd_object_ops[obj->type].destroy(obj);
99	112	iommufd_object_abort(ictx, obj);
100	113	}
101	114
@@ -223,6 +236,7 @@ static int iommufd_fops_open(struct inode inode, struct file filp) · 121 unmodified lines
223	236	}
224	237
225	238	xa_init_flags(&ictx->objects, XA_FLAGS_ALLOC1 \| XA_FLAGS_ACCOUNT);
	239	+ xa_init(&ictx->groups);
226	240	ictx->file = filp;
227	241	filp->private_data = ictx;
228	242	return 0;
@@ -258,6 +272,7 @@ static int iommufd_fops_release(struct inode inode, struct file filp) · 29 unmodified lines
258	272	if (WARN_ON(!destroyed))
259	273	break;
260	274	}
	275	+ WARN_ON(!xa_empty(&ictx->groups));
261	276	kfree(ictx);
262	277	return 0;
263	278	}
@@ -290,6 +305,8 @@ static int iommufd_option(struct iommufd_ucmd *ucmd) · 26 unmodified lines
290	305
291	306	union ucmd_buffer {
292	307	struct iommu_destroy destroy;
	308	+ struct iommu_hw_info info;
	309	+ struct iommu_hwpt_alloc hwpt;
293	310	struct iommu_ioas_alloc alloc;
294	311	struct iommu_ioas_allow_iovas allow_iovas;
295	312	struct iommu_ioas_copy ioas_copy;
@@ -321,6 +338,10 @@ struct iommufd_ioctl_op { · 25 unmodified lines
321	338	}
322	339	static const struct iommufd_ioctl_op iommufd_ioctl_ops[] = {
323	340	IOCTL_OP(IOMMU_DESTROY, iommufd_destroy, struct iommu_destroy, id),
	341	+ IOCTL_OP(IOMMU_GET_HW_INFO, iommufd_get_hw_info, struct iommu_hw_info,
	342	+ __reserved),
	343	+ IOCTL_OP(IOMMU_HWPT_ALLOC, iommufd_hwpt_alloc, struct iommu_hwpt_alloc,
	344	+ __reserved),
324	345	IOCTL_OP(IOMMU_IOAS_ALLOC, iommufd_ioas_alloc_ioctl,
325	346	struct iommu_ioas_alloc, out_ioas_id),
326	347	IOCTL_OP(IOMMU_IOAS_ALLOW_IOVAS, iommufd_ioas_allow_iovas,
@@ -463,6 +484,7 @@ static const struct iommufd_object_ops iommufd_object_ops[] = { · 136 unmodified lines
463	484	},
464	485	[IOMMUFD_OBJ_HW_PAGETABLE] = {
465	486	.destroy = iommufd_hw_pagetable_destroy,
	487	+ .abort = iommufd_hw_pagetable_abort,
466	488	},
467	489	#ifdef CONFIG_IOMMUFD_TEST
468	490	[IOMMUFD_OBJ_SELFTEST] = {
@@ -501,8 +523,14 @@ static int __init iommufd_init(void) · 32 unmodified lines
501	523	if (ret)
502	524	goto err_misc;
503	525	}
504		− iommufd_test_init();
	526	+ ret = iommufd_test_init();
	527	+ if (ret)
	528	+ goto err_vfio_misc;
505	529	return 0;
	530	+
	531	+err_vfio_misc:
	532	+ if (IS_ENABLED(CONFIG_IOMMUFD_VFIO_CONTAINER))
	533	+ misc_deregister(&vfio_misc_dev);
506	534	err_misc:
507	535	misc_deregister(&iommu_misc_dev);
508	536	return ret;
@@ -523,5 +551,6 @@ module_exit(iommufd_exit); · 14 unmodified lines
523	551	MODULE_ALIAS_MISCDEV(VFIO_MINOR);
524	552	MODULE_ALIAS("devname:vfio/vfio");
525	553	#endif
	554	+MODULE_IMPORT_NS(IOMMUFD_INTERNAL);
526	555	MODULE_DESCRIPTION("I/O Address Space Management for passthrough devices");
527	556	MODULE_LICENSE("GPL");

drivers/iommu/iommufd/selftest.c+157−56

@@ -9,14 +9,17 @@ · 8 unmodified lines
9	9	#include <linux/file.h>
10	10	#include <linux/anon_inodes.h>
11	11	#include <linux/fault-inject.h>
	12	+#include <linux/platform_device.h>
12	13	#include <uapi/linux/iommufd.h>
13	14
	15	+#include "../iommu-priv.h"
14	16	#include "io_pagetable.h"
15	17	#include "iommufd_private.h"
16	18	#include "iommufd_test.h"
17	19
18	20	static DECLARE_FAULT_ATTR(fail_iommufd);
19	21	static struct dentry *dbgfs_root;
	22	+static struct platform_device *selftest_iommu_dev;
20	23
21	24	size_t iommufd_test_memory_limit = 65536;
22	25
@@ -128,6 +131,21 @@ static struct iommu_domain mock_blocking_domain = { · 105 unmodified lines
128	131	.ops = &mock_blocking_ops,
129	132	};
130	133
	134	+static void mock_domain_hw_info(struct device dev, u32 length, u32 type)
	135	+{
	136	+ struct iommu_test_hw_info *info;
	137	+
	138	+ info = kzalloc(sizeof(*info), GFP_KERNEL);
	139	+ if (!info)
	140	+ return ERR_PTR(-ENOMEM);
	141	+
	142	+ info->test_reg = IOMMU_HW_INFO_SELFTEST_REGVAL;
	143	+ length = sizeof(info);
	144	+ *type = IOMMU_HW_INFO_TYPE_SELFTEST;
	145	+
	146	+ return info;
	147	+}
	148	+
131	149	static struct iommu_domain *mock_domain_alloc(unsigned int iommu_domain_type)
132	150	{
133	151	struct mock_iommu_domain *mock;
@@ -135,7 +153,7 @@ static struct iommu_domain *mock_domain_alloc(unsigned int iommu_domain_type) · 1 unmodified line
135	153	if (iommu_domain_type == IOMMU_DOMAIN_BLOCKED)
136	154	return &mock_blocking_domain;
137	155
138		− if (WARN_ON(iommu_domain_type != IOMMU_DOMAIN_UNMANAGED))
	156	+ if (iommu_domain_type != IOMMU_DOMAIN_UNMANAGED)
139	157	return NULL;
140	158
141	159	mock = kzalloc(sizeof(*mock), GFP_KERNEL);
@@ -276,12 +294,23 @@ static void mock_domain_set_plaform_dma_ops(struct device *dev) · 134 unmodified lines
276	294	*/
277	295	}
278	296
	297	+static struct iommu_device mock_iommu_device = {
	298	+};
	299	+
	300	+static struct iommu_device mock_probe_device(struct device dev)
	301	+{
	302	+ return &mock_iommu_device;
	303	+}
	304	+
279	305	static const struct iommu_ops mock_ops = {
280	306	.owner = THIS_MODULE,
281	307	.pgsize_bitmap = MOCK_IO_PAGE_SIZE,
	308	+ .hw_info = mock_domain_hw_info,
282	309	.domain_alloc = mock_domain_alloc,
283	310	.capable = mock_domain_capable,
284	311	.set_platform_dma_ops = mock_domain_set_plaform_dma_ops,
	312	+ .device_group = generic_device_group,
	313	+ .probe_device = mock_probe_device,
285	314	.default_domain_ops =
286	315	&(struct iommu_domain_ops){
287	316	.free = mock_domain_free,
@@ -292,10 +321,6 @@ static const struct iommu_ops mock_ops = { · 4 unmodified lines
292	321	},
293	322	};
294	323
295		−static struct iommu_device mock_iommu_device = {
296		− .ops = &mock_ops,
297		−};
298		−
299	324	static inline struct iommufd_hw_pagetable *
300	325	get_md_pagetable(struct iommufd_ucmd *ucmd, u32 mockpt_id,
301	326	struct mock_iommu_domain **mock)
@@ -316,22 +341,29 @@ get_md_pagetable(struct iommufd_ucmd *ucmd, u32 mockpt_id, · 14 unmodified lines
316	341	return hwpt;
317	342	}
318	343
319		−static struct bus_type iommufd_mock_bus_type = {
320		− .name = "iommufd_mock",
321		− .iommu_ops = &mock_ops,
	344	+struct mock_bus_type {
	345	+ struct bus_type bus;
	346	+ struct notifier_block nb;
	347	+};
	348	+
	349	+static struct mock_bus_type iommufd_mock_bus_type = {
	350	+ .bus = {
	351	+ .name = "iommufd_mock",
	352	+ },
322	353	};
323	354
	355	+static atomic_t mock_dev_num;
	356	+
324	357	static void mock_dev_release(struct device *dev)
325	358	{
326	359	struct mock_dev *mdev = container_of(dev, struct mock_dev, dev);
327	360
	361	+ atomic_dec(&mock_dev_num);
328	362	kfree(mdev);
329	363	}
330	364
331	365	static struct mock_dev *mock_dev_create(void)
332	366	{
333		− struct iommu_group *iommu_group;
334		− struct dev_iommu *dev_iommu;
335	367	struct mock_dev *mdev;
336	368	int rc;
337	369
@@ -341,51 +373,18 @@ static struct mock_dev *mock_dev_create(void) · 3 unmodified lines
341	373
342	374	device_initialize(&mdev->dev);
343	375	mdev->dev.release = mock_dev_release;
344		− mdev->dev.bus = &iommufd_mock_bus_type;
345		−
346		− iommu_group = iommu_group_alloc();
347		− if (IS_ERR(iommu_group)) {
348		− rc = PTR_ERR(iommu_group);
349		− goto err_put;
350		− }
	376	+ mdev->dev.bus = &iommufd_mock_bus_type.bus;
351	377
352	378	rc = dev_set_name(&mdev->dev, "iommufd_mock%u",
353		− iommu_group_id(iommu_group));
	379	+ atomic_inc_return(&mock_dev_num));
354	380	if (rc)
355		− goto err_group;
356		−
357		− /*
358		− * The iommu core has no way to associate a single device with an iommu
359		− * driver (heck currently it can't even support two iommu_drivers
360		− * registering). Hack it together with an open coded dev_iommu_get().
361		− * Notice that the normal notifier triggered iommu release process also
362		− * does not work here because this bus is not in iommu_buses.
363		− */
364		− mdev->dev.iommu = kzalloc(sizeof(*dev_iommu), GFP_KERNEL);
365		− if (!mdev->dev.iommu) {
366		− rc = -ENOMEM;
367		− goto err_group;
368		− }
369		− mutex_init(&mdev->dev.iommu->lock);
370		− mdev->dev.iommu->iommu_dev = &mock_iommu_device;
	381	+ goto err_put;
371	382
372	383	rc = device_add(&mdev->dev);
373	384	if (rc)
374		− goto err_dev_iommu;
375		−
376		− rc = iommu_group_add_device(iommu_group, &mdev->dev);
377		− if (rc)
378		− goto err_del;
379		− iommu_group_put(iommu_group);
	385	+ goto err_put;
380	386	return mdev;
381	387
382		−err_del:
383		− device_del(&mdev->dev);
384		−err_dev_iommu:
385		− kfree(mdev->dev.iommu);
386		− mdev->dev.iommu = NULL;
387		−err_group:
388		− iommu_group_put(iommu_group);
389	388	err_put:
390	389	put_device(&mdev->dev);
391	390	return ERR_PTR(rc);
@@ -393,11 +392,7 @@ err_put: · 1 unmodified line
393	392
394	393	static void mock_dev_destroy(struct mock_dev *mdev)
395	394	{
396		− iommu_group_remove_device(&mdev->dev);
397		− device_del(&mdev->dev);
398		− kfree(mdev->dev.iommu);
399		− mdev->dev.iommu = NULL;
400		− put_device(&mdev->dev);
	395	+ device_unregister(&mdev->dev);
401	396	}
402	397
403	398	bool iommufd_selftest_is_mock_dev(struct device *dev)
@@ -443,9 +438,15 @@ static int iommufd_test_mock_domain(struct iommufd_ucmd *ucmd, · 39 unmodified lines
443	438	/* Userspace must destroy the device_id to destroy the object */
444	439	cmd->mock_domain.out_hwpt_id = pt_id;
445	440	cmd->mock_domain.out_stdev_id = sobj->obj.id;
	441	+ cmd->mock_domain.out_idev_id = idev_id;
	442	+ rc = iommufd_ucmd_respond(ucmd, sizeof(*cmd));
	443	+ if (rc)
	444	+ goto out_detach;
446	445	iommufd_object_finalize(ucmd->ictx, &sobj->obj);
447		− return iommufd_ucmd_respond(ucmd, sizeof(*cmd));
	446	+ return 0;
448	447
	448	+out_detach:
	449	+ iommufd_device_detach(idev);
449	450	out_unbind:
450	451	iommufd_device_unbind(idev);
451	452	out_mdev:
@@ -455,6 +456,42 @@ out_sobj: · 3 unmodified lines
455	456	return rc;
456	457	}
457	458
	459	+/* Replace the mock domain with a manually allocated hw_pagetable */
	460	+static int iommufd_test_mock_domain_replace(struct iommufd_ucmd *ucmd,
	461	+ unsigned int device_id, u32 pt_id,
	462	+ struct iommu_test_cmd *cmd)
	463	+{
	464	+ struct iommufd_object *dev_obj;
	465	+ struct selftest_obj *sobj;
	466	+ int rc;
	467	+
	468	+ /*
	469	+ * Prefer to use the OBJ_SELFTEST because the destroy_rwsem will ensure
	470	+ * it doesn't race with detach, which is not allowed.
	471	+ */
	472	+ dev_obj =
	473	+ iommufd_get_object(ucmd->ictx, device_id, IOMMUFD_OBJ_SELFTEST);
	474	+ if (IS_ERR(dev_obj))
	475	+ return PTR_ERR(dev_obj);
	476	+
	477	+ sobj = container_of(dev_obj, struct selftest_obj, obj);
	478	+ if (sobj->type != TYPE_IDEV) {
	479	+ rc = -EINVAL;
	480	+ goto out_dev_obj;
	481	+ }
	482	+
	483	+ rc = iommufd_device_replace(sobj->idev.idev, &pt_id);
	484	+ if (rc)
	485	+ goto out_dev_obj;
	486	+
	487	+ cmd->mock_domain_replace.pt_id = pt_id;
	488	+ rc = iommufd_ucmd_respond(ucmd, sizeof(*cmd));
	489	+
	490	+out_dev_obj:
	491	+ iommufd_put_object(dev_obj);
	492	+ return rc;
	493	+}
	494	+
458	495	/* Add an additional reserved IOVA to the IOAS */
459	496	static int iommufd_test_add_reserved(struct iommufd_ucmd *ucmd,
460	497	unsigned int mockpt_id,
@@ -748,6 +785,22 @@ out_free_staccess: · 287 unmodified lines
748	785	return rc;
749	786	}
750	787
	788	+static int iommufd_test_access_replace_ioas(struct iommufd_ucmd *ucmd,
	789	+ unsigned int access_id,
	790	+ unsigned int ioas_id)
	791	+{
	792	+ struct selftest_access *staccess;
	793	+ int rc;
	794	+
	795	+ staccess = iommufd_access_get(access_id);
	796	+ if (IS_ERR(staccess))
	797	+ return PTR_ERR(staccess);
	798	+
	799	+ rc = iommufd_access_replace(staccess->access, ioas_id);
	800	+ fput(staccess->file);
	801	+ return rc;
	802	+}
	803	+
751	804	/* Check that the pages in a page array match the pages in the user VA */
752	805	static int iommufd_test_check_pages(void __user uptr, struct page *pages,
753	806	size_t npages)
@@ -948,6 +1001,9 @@ int iommufd_test(struct iommufd_ucmd *ucmd) · 194 unmodified lines
948	1001	cmd->add_reserved.length);
949	1002	case IOMMU_TEST_OP_MOCK_DOMAIN:
950	1003	return iommufd_test_mock_domain(ucmd, cmd);
	1004	+ case IOMMU_TEST_OP_MOCK_DOMAIN_REPLACE:
	1005	+ return iommufd_test_mock_domain_replace(
	1006	+ ucmd, cmd->id, cmd->mock_domain_replace.pt_id, cmd);
951	1007	case IOMMU_TEST_OP_MD_CHECK_MAP:
952	1008	return iommufd_test_md_check_pa(
953	1009	ucmd, cmd->id, cmd->check_map.iova,
@@ -960,6 +1016,9 @@ int iommufd_test(struct iommufd_ucmd *ucmd) · 6 unmodified lines
960	1016	case IOMMU_TEST_OP_CREATE_ACCESS:
961	1017	return iommufd_test_create_access(ucmd, cmd->id,
962	1018	cmd->create_access.flags);
	1019	+ case IOMMU_TEST_OP_ACCESS_REPLACE_IOAS:
	1020	+ return iommufd_test_access_replace_ioas(
	1021	+ ucmd, cmd->id, cmd->access_replace_ioas.ioas_id);
963	1022	case IOMMU_TEST_OP_ACCESS_PAGES:
964	1023	return iommufd_test_access_pages(
965	1024	ucmd, cmd->id, cmd->access_pages.iova,
@@ -992,15 +1051,57 @@ bool iommufd_should_fail(void) · 26 unmodified lines
992	1051	return should_fail(&fail_iommufd, 1);
993	1052	}
994	1053
995		−void __init iommufd_test_init(void)
	1054	+int __init iommufd_test_init(void)
996	1055	{
	1056	+ struct platform_device_info pdevinfo = {
	1057	+ .name = "iommufd_selftest_iommu",
	1058	+ };
	1059	+ int rc;
	1060	+
997	1061	dbgfs_root =
998	1062	fault_create_debugfs_attr("fail_iommufd", NULL, &fail_iommufd);
999		− WARN_ON(bus_register(&iommufd_mock_bus_type));
	1063	+
	1064	+ selftest_iommu_dev = platform_device_register_full(&pdevinfo);
	1065	+ if (IS_ERR(selftest_iommu_dev)) {
	1066	+ rc = PTR_ERR(selftest_iommu_dev);
	1067	+ goto err_dbgfs;
	1068	+ }
	1069	+
	1070	+ rc = bus_register(&iommufd_mock_bus_type.bus);
	1071	+ if (rc)
	1072	+ goto err_platform;
	1073	+
	1074	+ rc = iommu_device_sysfs_add(&mock_iommu_device,
	1075	+ &selftest_iommu_dev->dev, NULL, "%s",
	1076	+ dev_name(&selftest_iommu_dev->dev));
	1077	+ if (rc)
	1078	+ goto err_bus;
	1079	+
	1080	+ rc = iommu_device_register_bus(&mock_iommu_device, &mock_ops,
	1081	+ &iommufd_mock_bus_type.bus,
	1082	+ &iommufd_mock_bus_type.nb);
	1083	+ if (rc)
	1084	+ goto err_sysfs;
	1085	+ return 0;
	1086	+
	1087	+err_sysfs:
	1088	+ iommu_device_sysfs_remove(&mock_iommu_device);
	1089	+err_bus:
	1090	+ bus_unregister(&iommufd_mock_bus_type.bus);
	1091	+err_platform:
	1092	+ platform_device_unregister(selftest_iommu_dev);
	1093	+err_dbgfs:
	1094	+ debugfs_remove_recursive(dbgfs_root);
	1095	+ return rc;
1000	1096	}
1001	1097
1002	1098	void iommufd_test_exit(void)
1003	1099	{
	1100	+ iommu_device_sysfs_remove(&mock_iommu_device);
	1101	+ iommu_device_unregister_bus(&mock_iommu_device,
	1102	+ &iommufd_mock_bus_type.bus,
	1103	+ &iommufd_mock_bus_type.nb);
	1104	+ bus_unregister(&iommufd_mock_bus_type.bus);
	1105	+ platform_device_unregister(selftest_iommu_dev);
1004	1106	debugfs_remove_recursive(dbgfs_root);
1005		− bus_unregister(&iommufd_mock_bus_type);
1006	1107	}

drivers/vfio/iommufd.c+6−5

@@ -146,9 +146,9 @@ int vfio_iommufd_physical_attach_ioas(struct vfio_device vdev, u32 pt_id) · 145 unmodified lines
146	146	return -EINVAL;
147	147
148	148	if (vdev->iommufd_attached)
149		− return -EBUSY;
150		−
151		− rc = iommufd_device_attach(vdev->iommufd_device, pt_id);
	149	+ rc = iommufd_device_replace(vdev->iommufd_device, pt_id);
	150	+ else
	151	+ rc = iommufd_device_attach(vdev->iommufd_device, pt_id);
152	152	if (rc)
153	153	return rc;
154	154	vdev->iommufd_attached = true;
@@ -223,8 +223,9 @@ int vfio_iommufd_emulated_attach_ioas(struct vfio_device vdev, u32 pt_id) · 68 unmodified lines
223	223	lockdep_assert_held(&vdev->dev_set->lock);
224	224
225	225	if (vdev->iommufd_attached)
226		− return -EBUSY;
227		− rc = iommufd_access_attach(vdev->iommufd_access, *pt_id);
	226	+ rc = iommufd_access_replace(vdev->iommufd_access, *pt_id);
	227	+ else
	228	+ rc = iommufd_access_attach(vdev->iommufd_access, *pt_id);
228	229	if (rc)
229	230	return rc;
230	231	vdev->iommufd_attached = true;

drivers/vfio/vfio_main.c+4−0

@@ -1536,6 +1536,8 @@ int vfio_pin_pages(struct vfio_device *device, dma_addr_t iova, · 1535 unmodified lines
1536	1536	/* group->container cannot change while a vfio device is open */
1537	1537	if (!pages \|\| !npage \|\| WARN_ON(!vfio_assert_device_open(device)))
1538	1538	return -EINVAL;
	1539	+ if (!device->ops->dma_unmap)
	1540	+ return -EINVAL;
1539	1541	if (vfio_device_has_container(device))
1540	1542	return vfio_device_container_pin_pages(device, iova,
1541	1543	npage, prot, pages);
@@ -1573,6 +1575,8 @@ void vfio_unpin_pages(struct vfio_device *device, dma_addr_t iova, int npage) · 31 unmodified lines
1573	1575	{
1574	1576	if (WARN_ON(!vfio_assert_device_open(device)))
1575	1577	return;
	1578	+ if (WARN_ON(!device->ops->dma_unmap))
	1579	+ return;
1576	1580
1577	1581	if (vfio_device_has_container(device)) {
1578	1582	vfio_device_container_unpin_pages(device, iova, npage);

include/linux/iommu.h+5−11

@@ -228,6 +228,10 @@ struct iommu_iotlb_gather { · 227 unmodified lines
228	228	/**
229	229	* struct iommu_ops - iommu ops and capabilities
230	230	* @capable: check capability
	231	+ * @hw_info: report iommu hardware information. The data buffer returned by this
	232	+ * op is allocated in the iommu driver and freed by the caller after
	233	+ * use. The information type is one of enum iommu_hw_info_type defined
	234	+ * in include/uapi/linux/iommufd.h.
231	235	* @domain_alloc: allocate iommu domain
232	236	* @probe_device: Add device to iommu driver handling
233	237	* @release_device: Remove device from iommu driver handling
@@ -257,6 +261,7 @@ struct iommu_iotlb_gather { · 23 unmodified lines
257	261	*/
258	262	struct iommu_ops {
259	263	bool (capable)(struct device dev, enum iommu_cap);
	264	+ void (hw_info)(struct device dev, u32 length, u32 *type);
260	265
261	266	/* Domain allocation and freeing by the iommu driver */
262	267	struct iommu_domain (domain_alloc)(unsigned iommu_domain_type);
@@ -450,17 +455,6 @@ static inline void iommu_iotlb_gather_init(struct iommu_iotlb_gather *gather) · 187 unmodified lines
450	455	};
451	456	}
452	457
453		−static inline const struct iommu_ops dev_iommu_ops(struct device dev)
454		−{
455		− /*
456		− * Assume that valid ops must be installed if iommu_probe_device()
457		− * has succeeded. The device ops are essentially for internal use
458		− * within the IOMMU subsystem itself, so we should be able to trust
459		− * ourselves not to misuse the helper.
460		− */
461		− return dev->iommu->iommu_dev->ops;
462		−}
463		−
464	458	extern int bus_iommu_probe(const struct bus_type *bus);
465	459	extern bool iommu_present(const struct bus_type *bus);
466	460	extern bool device_iommu_capable(struct device *dev, enum iommu_cap cap);

include/linux/iommufd.h+2−0

@@ -23,6 +23,7 @@ struct iommufd_device iommufd_device_bind(struct iommufd_ctx ictx, · 22 unmodified lines
23	23	void iommufd_device_unbind(struct iommufd_device *idev);
24	24
25	25	int iommufd_device_attach(struct iommufd_device idev, u32 pt_id);
	26	+int iommufd_device_replace(struct iommufd_device idev, u32 pt_id);
26	27	void iommufd_device_detach(struct iommufd_device *idev);
27	28
28	29	struct iommufd_ctx iommufd_device_to_ictx(struct iommufd_device idev);
@@ -48,6 +49,7 @@ iommufd_access_create(struct iommufd_ctx *ictx, · 19 unmodified lines
48	49	const struct iommufd_access_ops ops, void data, u32 *id);
49	50	void iommufd_access_destroy(struct iommufd_access *access);
50	51	int iommufd_access_attach(struct iommufd_access *access, u32 ioas_id);
	52	+int iommufd_access_replace(struct iommufd_access *access, u32 ioas_id);
51	53	void iommufd_access_detach(struct iommufd_access *access);
52	54
53	55	void iommufd_ctx_get(struct iommufd_ctx *ictx);

include/uapi/linux/iommufd.h+97−0

@@ -45,6 +45,8 @@ enum { · 44 unmodified lines
45	45	IOMMUFD_CMD_IOAS_UNMAP,
46	46	IOMMUFD_CMD_OPTION,
47	47	IOMMUFD_CMD_VFIO_IOAS,
	48	+ IOMMUFD_CMD_HWPT_ALLOC,
	49	+ IOMMUFD_CMD_GET_HW_INFO,
48	50	};
49	51
50	52	/**
@@ -344,4 +346,99 @@ struct iommu_vfio_ioas { · 293 unmodified lines
344	346	__u16 __reserved;
345	347	};
346	348	#define IOMMU_VFIO_IOAS _IO(IOMMUFD_TYPE, IOMMUFD_CMD_VFIO_IOAS)
	349	+
	350	+/**
	351	+ * struct iommu_hwpt_alloc - ioctl(IOMMU_HWPT_ALLOC)
	352	+ * @size: sizeof(struct iommu_hwpt_alloc)
	353	+ * @flags: Must be 0
	354	+ * @dev_id: The device to allocate this HWPT for
	355	+ * @pt_id: The IOAS to connect this HWPT to
	356	+ * @out_hwpt_id: The ID of the new HWPT
	357	+ * @__reserved: Must be 0
	358	+ *
	359	+ * Explicitly allocate a hardware page table object. This is the same object
	360	+ * type that is returned by iommufd_device_attach() and represents the
	361	+ * underlying iommu driver's iommu_domain kernel object.
	362	+ *
	363	+ * A HWPT will be created with the IOVA mappings from the given IOAS.
	364	+ */
	365	+struct iommu_hwpt_alloc {
	366	+ __u32 size;
	367	+ __u32 flags;
	368	+ __u32 dev_id;
	369	+ __u32 pt_id;
	370	+ __u32 out_hwpt_id;
	371	+ __u32 __reserved;
	372	+};
	373	+#define IOMMU_HWPT_ALLOC _IO(IOMMUFD_TYPE, IOMMUFD_CMD_HWPT_ALLOC)
	374	+
	375	+/**
	376	+ * struct iommu_hw_info_vtd - Intel VT-d hardware information
	377	+ *
	378	+ * @flags: Must be 0
	379	+ * @__reserved: Must be 0
	380	+ *
	381	+ * @cap_reg: Value of Intel VT-d capability register defined in VT-d spec
	382	+ * section 11.4.2 Capability Register.
	383	+ * @ecap_reg: Value of Intel VT-d capability register defined in VT-d spec
	384	+ * section 11.4.3 Extended Capability Register.
	385	+ *
	386	+ * User needs to understand the Intel VT-d specification to decode the
	387	+ * register value.
	388	+ */
	389	+struct iommu_hw_info_vtd {
	390	+ __u32 flags;
	391	+ __u32 __reserved;
	392	+ __aligned_u64 cap_reg;
	393	+ __aligned_u64 ecap_reg;
	394	+};
	395	+
	396	+/**
	397	+ * enum iommu_hw_info_type - IOMMU Hardware Info Types
	398	+ * @IOMMU_HW_INFO_TYPE_NONE: Used by the drivers that do not report hardware
	399	+ * info
	400	+ * @IOMMU_HW_INFO_TYPE_INTEL_VTD: Intel VT-d iommu info type
	401	+ */
	402	+enum iommu_hw_info_type {
	403	+ IOMMU_HW_INFO_TYPE_NONE,
	404	+ IOMMU_HW_INFO_TYPE_INTEL_VTD,
	405	+};
	406	+
	407	+/**
	408	+ * struct iommu_hw_info - ioctl(IOMMU_GET_HW_INFO)
	409	+ * @size: sizeof(struct iommu_hw_info)
	410	+ * @flags: Must be 0
	411	+ * @dev_id: The device bound to the iommufd
	412	+ * @data_len: Input the length of a user buffer in bytes. Output the length of
	413	+ * data that kernel supports
	414	+ * @data_uptr: User pointer to a user-space buffer used by the kernel to fill
	415	+ * the iommu type specific hardware information data
	416	+ * @out_data_type: Output the iommu hardware info type as defined in the enum
	417	+ * iommu_hw_info_type.
	418	+ * @__reserved: Must be 0
	419	+ *
	420	+ * Query an iommu type specific hardware information data from an iommu behind
	421	+ * a given device that has been bound to iommufd. This hardware info data will
	422	+ * be used to sync capabilities between the virtual iommu and the physical
	423	+ * iommu, e.g. a nested translation setup needs to check the hardware info, so
	424	+ * a guest stage-1 page table can be compatible with the physical iommu.
	425	+ *
	426	+ * To capture an iommu type specific hardware information data, @data_uptr and
	427	+ * its length @data_len must be provided. Trailing bytes will be zeroed if the
	428	+ * user buffer is larger than the data that kernel has. Otherwise, kernel only
	429	+ * fills the buffer using the given length in @data_len. If the ioctl succeeds,
	430	+ * @data_len will be updated to the length that kernel actually supports,
	431	+ * @out_data_type will be filled to decode the data filled in the buffer
	432	+ * pointed by @data_uptr. Input @data_len == zero is allowed.
	433	+ */
	434	+struct iommu_hw_info {
	435	+ __u32 size;
	436	+ __u32 flags;
	437	+ __u32 dev_id;
	438	+ __u32 data_len;
	439	+ __aligned_u64 data_uptr;
	440	+ __u32 out_data_type;
	441	+ __u32 __reserved;
	442	+};
	443	+#define IOMMU_GET_HW_INFO _IO(IOMMUFD_TYPE, IOMMUFD_CMD_GET_HW_INFO)
347	444	#endif

include/uapi/linux/vfio.h+6−0

@@ -940,6 +940,12 @@ struct vfio_device_bind_iommufd { · 939 unmodified lines
940	940	* Undo by VFIO_DEVICE_DETACH_IOMMUFD_PT or device fd close. This is only
941	941	* allowed on cdev fds.
942	942	*
	943	+ * If a vfio device is currently attached to a valid hw_pagetable, without doing
	944	+ * a VFIO_DEVICE_DETACH_IOMMUFD_PT, a second VFIO_DEVICE_ATTACH_IOMMUFD_PT ioctl
	945	+ * passing in another hw_pagetable (hwpt) id is allowed. This action, also known
	946	+ * as a hw_pagetable replacement, will replace the device's currently attached
	947	+ * hw_pagetable with a new hw_pagetable corresponding to the given pt_id.
	948	+ *
943	949	* Return: 0 on success, -errno on failure.
944	950	*/
945	951	struct vfio_device_attach_iommufd_pt {

tools/testing/selftests/iommu/iommufd.c+118−12

@@ -9,9 +9,6 @@ · 8 unmodified lines
9	9
10	10	#include "iommufd_utils.h"
11	11
12		−static void *buffer;
13		−
14		−static unsigned long PAGE_SIZE;
15	12	static unsigned long HUGEPAGE_SIZE;
16	13
17	14	#define MOCK_PAGE_SIZE (PAGE_SIZE / 2)
@@ -116,6 +113,7 @@ TEST_F(iommufd, cmd_length) · 98 unmodified lines
116	113	}
117	114
118	115	TEST_LENGTH(iommu_destroy, IOMMU_DESTROY);
	116	+ TEST_LENGTH(iommu_hw_info, IOMMU_GET_HW_INFO);
119	117	TEST_LENGTH(iommu_ioas_alloc, IOMMU_IOAS_ALLOC);
120	118	TEST_LENGTH(iommu_ioas_iova_ranges, IOMMU_IOAS_IOVA_RANGES);
121	119	TEST_LENGTH(iommu_ioas_allow_iovas, IOMMU_IOAS_ALLOW_IOVAS);
@@ -188,6 +186,7 @@ FIXTURE(iommufd_ioas) · 66 unmodified lines
188	186	uint32_t ioas_id;
189	187	uint32_t stdev_id;
190	188	uint32_t hwpt_id;
	189	+ uint32_t device_id;
191	190	uint64_t base_iova;
192	191	};
193	192
@@ -214,7 +213,7 @@ FIXTURE_SETUP(iommufd_ioas) · 20 unmodified lines
214	213
215	214	for (i = 0; i != variant->mock_domains; i++) {
216	215	test_cmd_mock_domain(self->ioas_id, &self->stdev_id,
217		− &self->hwpt_id);
	216	+ &self->hwpt_id, &self->device_id);
218	217	self->base_iova = MOCK_APERTURE_START;
219	218	}
220	219	}
@@ -265,7 +264,7 @@ TEST_F(iommufd_ioas, hwpt_attach) · 44 unmodified lines
265	264	{
266	265	/* Create a device attached directly to a hwpt */
267	266	if (self->stdev_id) {
268		− test_cmd_mock_domain(self->hwpt_id, NULL, NULL);
	267	+ test_cmd_mock_domain(self->hwpt_id, NULL, NULL, NULL);
269	268	} else {
270	269	test_err_mock_domain(ENOENT, self->hwpt_id, NULL, NULL);
271	270	}
@@ -293,6 +292,40 @@ TEST_F(iommufd_ioas, ioas_area_auto_destroy) · 21 unmodified lines
293	292	}
294	293	}
295	294
	295	+TEST_F(iommufd_ioas, get_hw_info)
	296	+{
	297	+ struct iommu_test_hw_info buffer_exact;
	298	+ struct iommu_test_hw_info_buffer_larger {
	299	+ struct iommu_test_hw_info info;
	300	+ uint64_t trailing_bytes;
	301	+ } buffer_larger;
	302	+ struct iommu_test_hw_info_buffer_smaller {
	303	+ __u32 flags;
	304	+ } buffer_smaller;
	305	+
	306	+ if (self->device_id) {
	307	+ /* Provide a zero-size user_buffer */
	308	+ test_cmd_get_hw_info(self->device_id, NULL, 0);
	309	+ /* Provide a user_buffer with exact size */
	310	+ test_cmd_get_hw_info(self->device_id, &buffer_exact, sizeof(buffer_exact));
	311	+ /*
	312	+ * Provide a user_buffer with size larger than the exact size to check if
	313	+ * kernel zero the trailing bytes.
	314	+ */
	315	+ test_cmd_get_hw_info(self->device_id, &buffer_larger, sizeof(buffer_larger));
	316	+ /*
	317	+ * Provide a user_buffer with size smaller than the exact size to check if
	318	+ * the fields within the size range still gets updated.
	319	+ */
	320	+ test_cmd_get_hw_info(self->device_id, &buffer_smaller, sizeof(buffer_smaller));
	321	+ } else {
	322	+ test_err_get_hw_info(ENOENT, self->device_id,
	323	+ &buffer_exact, sizeof(buffer_exact));
	324	+ test_err_get_hw_info(ENOENT, self->device_id,
	325	+ &buffer_larger, sizeof(buffer_larger));
	326	+ }
	327	+}
	328	+
296	329	TEST_F(iommufd_ioas, area)
297	330	{
298	331	int i;
@@ -684,7 +717,7 @@ TEST_F(iommufd_ioas, access_pin) · 385 unmodified lines
684	717	_IOMMU_TEST_CMD(IOMMU_TEST_OP_ACCESS_PAGES),
685	718	&access_cmd));
686	719	test_cmd_mock_domain(self->ioas_id, &mock_stdev_id,
687		− &mock_hwpt_id);
	720	+ &mock_hwpt_id, NULL);
688	721	check_map_cmd.id = mock_hwpt_id;
689	722	ASSERT_EQ(0, ioctl(self->fd,
690	723	_IOMMU_TEST_CMD(IOMMU_TEST_OP_MD_CHECK_MAP),
@@ -839,7 +872,7 @@ TEST_F(iommufd_ioas, fork_gone) · 148 unmodified lines
839	872	* If a domain already existed then everything was pinned within
840	873	* the fork, so this copies from one domain to another.
841	874	*/
842		− test_cmd_mock_domain(self->ioas_id, NULL, NULL);
	875	+ test_cmd_mock_domain(self->ioas_id, NULL, NULL, NULL);
843	876	check_access_rw(_metadata, self->fd, access_id,
844	877	MOCK_APERTURE_START, 0);
845	878
@@ -888,7 +921,7 @@ TEST_F(iommufd_ioas, fork_present) · 42 unmodified lines
888	921	ASSERT_EQ(8, read(efd, &tmp, sizeof(tmp)));
889	922
890	923	/* Read pages from the remote process */
891		− test_cmd_mock_domain(self->ioas_id, NULL, NULL);
	924	+ test_cmd_mock_domain(self->ioas_id, NULL, NULL, NULL);
892	925	check_access_rw(_metadata, self->fd, access_id, MOCK_APERTURE_START, 0);
893	926
894	927	ASSERT_EQ(0, close(pipefds[1]));
@@ -1035,6 +1068,8 @@ FIXTURE(iommufd_mock_domain) · 140 unmodified lines
1035	1068	uint32_t ioas_id;
1036	1069	uint32_t hwpt_id;
1037	1070	uint32_t hwpt_ids[2];
	1071	+ uint32_t stdev_ids[2];
	1072	+ uint32_t idev_ids[2];
1038	1073	int mmap_flags;
1039	1074	size_t mmap_buf_size;
1040	1075	};
@@ -1056,7 +1091,8 @@ FIXTURE_SETUP(iommufd_mock_domain) · 15 unmodified lines
1056	1091	ASSERT_GE(ARRAY_SIZE(self->hwpt_ids), variant->mock_domains);
1057	1092
1058	1093	for (i = 0; i != variant->mock_domains; i++)
1059		− test_cmd_mock_domain(self->ioas_id, NULL, &self->hwpt_ids[i]);
	1094	+ test_cmd_mock_domain(self->ioas_id, &self->stdev_ids[i],
	1095	+ &self->hwpt_ids[i], &self->idev_ids[i]);
1060	1096	self->hwpt_id = self->hwpt_ids[0];
1061	1097
1062	1098	self->mmap_flags = MAP_SHARED \| MAP_ANONYMOUS;
@@ -1250,7 +1286,7 @@ TEST_F(iommufd_mock_domain, all_aligns_copy) · 187 unmodified lines
1250	1286	/* Add and destroy a domain while the area exists */
1251	1287	old_id = self->hwpt_ids[1];
1252	1288	test_cmd_mock_domain(self->ioas_id, &mock_stdev_id,
1253		− &self->hwpt_ids[1]);
	1289	+ &self->hwpt_ids[1], NULL);
1254	1290
1255	1291	check_mock_iova(buf + start, iova, length);
1256	1292	check_refs(buf + start / PAGE_SIZE * PAGE_SIZE,
@@ -1283,7 +1319,13 @@ TEST_F(iommufd_mock_domain, user_copy) · 26 unmodified lines
1283	1319	.dst_iova = MOCK_APERTURE_START,
1284	1320	.length = BUFFER_SIZE,
1285	1321	};
1286		− unsigned int ioas_id;
	1322	+ struct iommu_ioas_unmap unmap_cmd = {
	1323	+ .size = sizeof(unmap_cmd),
	1324	+ .ioas_id = self->ioas_id,
	1325	+ .iova = MOCK_APERTURE_START,
	1326	+ .length = BUFFER_SIZE,
	1327	+ };
	1328	+ unsigned int new_ioas_id, ioas_id;
1287	1329
1288	1330	/* Pin the pages in an IOAS with no domains then copy to an IOAS with domains */
1289	1331	test_ioctl_ioas_alloc(&ioas_id);
@@ -1301,13 +1343,77 @@ TEST_F(iommufd_mock_domain, user_copy) · 11 unmodified lines
1301	1343	ASSERT_EQ(0, ioctl(self->fd, IOMMU_IOAS_COPY, &copy_cmd));
1302	1344	check_mock_iova(buffer, MOCK_APERTURE_START, BUFFER_SIZE);
1303	1345
	1346	+ /* Now replace the ioas with a new one */
	1347	+ test_ioctl_ioas_alloc(&new_ioas_id);
	1348	+ test_ioctl_ioas_map_id(new_ioas_id, buffer, BUFFER_SIZE,
	1349	+ &copy_cmd.src_iova);
	1350	+ test_cmd_access_replace_ioas(access_cmd.id, new_ioas_id);
	1351	+
	1352	+ /* Destroy the old ioas and cleanup copied mapping */
	1353	+ ASSERT_EQ(0, ioctl(self->fd, IOMMU_IOAS_UNMAP, &unmap_cmd));
	1354	+ test_ioctl_destroy(ioas_id);
	1355	+
	1356	+ /* Then run the same test again with the new ioas */
	1357	+ access_cmd.access_pages.iova = copy_cmd.src_iova;
	1358	+ ASSERT_EQ(0,
	1359	+ ioctl(self->fd, _IOMMU_TEST_CMD(IOMMU_TEST_OP_ACCESS_PAGES),
	1360	+ &access_cmd));
	1361	+ copy_cmd.src_ioas_id = new_ioas_id;
	1362	+ ASSERT_EQ(0, ioctl(self->fd, IOMMU_IOAS_COPY, &copy_cmd));
	1363	+ check_mock_iova(buffer, MOCK_APERTURE_START, BUFFER_SIZE);
	1364	+
1304	1365	test_cmd_destroy_access_pages(
1305	1366	access_cmd.id, access_cmd.access_pages.out_access_pages_id);
1306	1367	test_cmd_destroy_access(access_cmd.id);
1307	1368
	1369	+ test_ioctl_destroy(new_ioas_id);
	1370	+}
	1371	+
	1372	+TEST_F(iommufd_mock_domain, replace)
	1373	+{
	1374	+ uint32_t ioas_id;
	1375	+
	1376	+ test_ioctl_ioas_alloc(&ioas_id);
	1377	+
	1378	+ test_cmd_mock_domain_replace(self->stdev_ids[0], ioas_id);
	1379	+
	1380	+ /*
	1381	+ * Replacing the IOAS causes the prior HWPT to be deallocated, thus we
	1382	+ * should get enoent when we try to use it.
	1383	+ */
	1384	+ if (variant->mock_domains == 1)
	1385	+ test_err_mock_domain_replace(ENOENT, self->stdev_ids[0],
	1386	+ self->hwpt_ids[0]);
	1387	+
	1388	+ test_cmd_mock_domain_replace(self->stdev_ids[0], ioas_id);
	1389	+ if (variant->mock_domains >= 2) {
	1390	+ test_cmd_mock_domain_replace(self->stdev_ids[0],
	1391	+ self->hwpt_ids[1]);
	1392	+ test_cmd_mock_domain_replace(self->stdev_ids[0],
	1393	+ self->hwpt_ids[1]);
	1394	+ test_cmd_mock_domain_replace(self->stdev_ids[0],
	1395	+ self->hwpt_ids[0]);
	1396	+ }
	1397	+
	1398	+ test_cmd_mock_domain_replace(self->stdev_ids[0], self->ioas_id);
1308	1399	test_ioctl_destroy(ioas_id);
1309	1400	}
1310	1401
	1402	+TEST_F(iommufd_mock_domain, alloc_hwpt)
	1403	+{
	1404	+ int i;
	1405	+
	1406	+ for (i = 0; i != variant->mock_domains; i++) {
	1407	+ uint32_t stddev_id;
	1408	+ uint32_t hwpt_id;
	1409	+
	1410	+ test_cmd_hwpt_alloc(self->idev_ids[0], self->ioas_id, &hwpt_id);
	1411	+ test_cmd_mock_domain(hwpt_id, &stddev_id, NULL, NULL);
	1412	+ test_ioctl_destroy(stddev_id);
	1413	+ test_ioctl_destroy(hwpt_id);
	1414	+ }
	1415	+}
	1416	+
1311	1417	/* VFIO compatibility IOCTLs */
1312	1418
1313	1419	TEST_F(iommufd, simple_ioctls)
@@ -1429,7 +1535,7 @@ FIXTURE_SETUP(vfio_compat_mock_domain) · 115 unmodified lines
1429	1535
1430	1536	/* Create what VFIO would consider a group */
1431	1537	test_ioctl_ioas_alloc(&self->ioas_id);
1432		− test_cmd_mock_domain(self->ioas_id, NULL, NULL);
	1538	+ test_cmd_mock_domain(self->ioas_id, NULL, NULL, NULL);
1433	1539
1434	1540	/* Attach it to the vfio compat */
1435	1541	vfio_ioas_cmd.ioas_id = self->ioas_id;

tools/testing/selftests/iommu/iommufd_fail_nth.c+64−7

@@ -41,6 +41,8 @@ static int writeat(int dfd, const char fn, const char val) · 40 unmodified lines
41	41
42	42	static __attribute__((constructor)) void setup_buffer(void)
43	43	{
	44	+ PAGE_SIZE = sysconf(_SC_PAGE_SIZE);
	45	+
44	46	BUFFER_SIZE = 210241024;
45	47
46	48	buffer = mmap(0, BUFFER_SIZE, PROT_READ \| PROT_WRITE,
@@ -313,7 +315,7 @@ TEST_FAIL_NTH(basic_fail_nth, map_domain) · 266 unmodified lines
313	315
314	316	fail_nth_enable();
315	317
316		− if (_test_cmd_mock_domain(self->fd, ioas_id, &stdev_id, &hwpt_id))
	318	+ if (_test_cmd_mock_domain(self->fd, ioas_id, &stdev_id, &hwpt_id, NULL))
317	319	return -1;
318	320
319	321	if (_test_ioctl_ioas_map(self->fd, ioas_id, buffer, 262144, &iova,
@@ -324,7 +326,7 @@ TEST_FAIL_NTH(basic_fail_nth, map_domain) · 4 unmodified lines
324	326	if (_test_ioctl_destroy(self->fd, stdev_id))
325	327	return -1;
326	328
327		− if (_test_cmd_mock_domain(self->fd, ioas_id, &stdev_id, &hwpt_id))
	329	+ if (_test_cmd_mock_domain(self->fd, ioas_id, &stdev_id, &hwpt_id, NULL))
328	330	return -1;
329	331	return 0;
330	332	}
@@ -348,12 +350,13 @@ TEST_FAIL_NTH(basic_fail_nth, map_two_domains) · 17 unmodified lines
348	350	if (_test_ioctl_set_temp_memory_limit(self->fd, 32))
349	351	return -1;
350	352
351		− if (_test_cmd_mock_domain(self->fd, ioas_id, &stdev_id, &hwpt_id))
	353	+ if (_test_cmd_mock_domain(self->fd, ioas_id, &stdev_id, &hwpt_id, NULL))
352	354	return -1;
353	355
354	356	fail_nth_enable();
355	357
356		− if (_test_cmd_mock_domain(self->fd, ioas_id, &stdev_id2, &hwpt_id2))
	358	+ if (_test_cmd_mock_domain(self->fd, ioas_id, &stdev_id2, &hwpt_id2,
	359	+ NULL))
357	360	return -1;
358	361
359	362	if (_test_ioctl_ioas_map(self->fd, ioas_id, buffer, 262144, &iova,
@@ -367,9 +370,10 @@ TEST_FAIL_NTH(basic_fail_nth, map_two_domains) · 7 unmodified lines
367	370	if (_test_ioctl_destroy(self->fd, stdev_id2))
368	371	return -1;
369	372
370		− if (_test_cmd_mock_domain(self->fd, ioas_id, &stdev_id, &hwpt_id))
	373	+ if (_test_cmd_mock_domain(self->fd, ioas_id, &stdev_id, &hwpt_id, NULL))
371	374	return -1;
372		− if (_test_cmd_mock_domain(self->fd, ioas_id, &stdev_id2, &hwpt_id2))
	375	+ if (_test_cmd_mock_domain(self->fd, ioas_id, &stdev_id2, &hwpt_id2,
	376	+ NULL))
373	377	return -1;
374	378	return 0;
375	379	}
@@ -526,7 +530,7 @@ TEST_FAIL_NTH(basic_fail_nth, access_pin_domain) · 150 unmodified lines
526	530	if (_test_ioctl_set_temp_memory_limit(self->fd, 32))
527	531	return -1;
528	532
529		− if (_test_cmd_mock_domain(self->fd, ioas_id, &stdev_id, &hwpt_id))
	533	+ if (_test_cmd_mock_domain(self->fd, ioas_id, &stdev_id, &hwpt_id, NULL))
530	534	return -1;
531	535
532	536	if (_test_ioctl_ioas_map(self->fd, ioas_id, buffer, BUFFER_SIZE, &iova,
@@ -569,4 +573,57 @@ TEST_FAIL_NTH(basic_fail_nth, access_pin_domain) · 36 unmodified lines
569	573	return 0;
570	574	}
571	575
	576	+/* device.c */
	577	+TEST_FAIL_NTH(basic_fail_nth, device)
	578	+{
	579	+ struct iommu_test_hw_info info;
	580	+ uint32_t ioas_id;
	581	+ uint32_t ioas_id2;
	582	+ uint32_t stdev_id;
	583	+ uint32_t idev_id;
	584	+ uint32_t hwpt_id;
	585	+ __u64 iova;
	586	+
	587	+ self->fd = open("/dev/iommu", O_RDWR);
	588	+ if (self->fd == -1)
	589	+ return -1;
	590	+
	591	+ if (_test_ioctl_ioas_alloc(self->fd, &ioas_id))
	592	+ return -1;
	593	+
	594	+ if (_test_ioctl_ioas_alloc(self->fd, &ioas_id2))
	595	+ return -1;
	596	+
	597	+ iova = MOCK_APERTURE_START;
	598	+ if (_test_ioctl_ioas_map(self->fd, ioas_id, buffer, PAGE_SIZE, &iova,
	599	+ IOMMU_IOAS_MAP_FIXED_IOVA \|
	600	+ IOMMU_IOAS_MAP_WRITEABLE \|
	601	+ IOMMU_IOAS_MAP_READABLE))
	602	+ return -1;
	603	+ if (_test_ioctl_ioas_map(self->fd, ioas_id2, buffer, PAGE_SIZE, &iova,
	604	+ IOMMU_IOAS_MAP_FIXED_IOVA \|
	605	+ IOMMU_IOAS_MAP_WRITEABLE \|
	606	+ IOMMU_IOAS_MAP_READABLE))
	607	+ return -1;
	608	+
	609	+ fail_nth_enable();
	610	+
	611	+ if (_test_cmd_mock_domain(self->fd, ioas_id, &stdev_id, NULL,
	612	+ &idev_id))
	613	+ return -1;
	614	+
	615	+ if (_test_cmd_get_hw_info(self->fd, idev_id, &info, sizeof(info)))
	616	+ return -1;
	617	+
	618	+ if (_test_cmd_hwpt_alloc(self->fd, idev_id, ioas_id, &hwpt_id))
	619	+ return -1;
	620	+
	621	+ if (_test_cmd_mock_domain_replace(self->fd, stdev_id, ioas_id2, NULL))
	622	+ return -1;
	623	+
	624	+ if (_test_cmd_mock_domain_replace(self->fd, stdev_id, hwpt_id, NULL))
	625	+ return -1;
	626	+ return 0;
	627	+}
	628	+
572	629	TEST_HARNESS_MAIN

tools/testing/selftests/iommu/iommufd_utils.h+139−5

@@ -19,6 +19,12 @@ · 18 unmodified lines
19	19	static void *buffer;
20	20	static unsigned long BUFFER_SIZE;
21	21
	22	+static unsigned long PAGE_SIZE;
	23	+
	24	+#define sizeof_field(TYPE, MEMBER) sizeof((((TYPE *)0)->MEMBER))
	25	+#define offsetofend(TYPE, MEMBER) \
	26	+ (offsetof(TYPE, MEMBER) + sizeof_field(TYPE, MEMBER))
	27	+
22	28	/*
23	29	* Have the kernel check the refcount on pages. I don't know why a freshly
24	30	* mmap'd anon non-compound page starts out with a ref of 3
@@ -39,7 +45,7 @@ static unsigned long BUFFER_SIZE; · 14 unmodified lines
39	45	})
40	46
41	47	static int _test_cmd_mock_domain(int fd, unsigned int ioas_id, __u32 *stdev_id,
42		− __u32 *hwpt_id)
	48	+ __u32 hwpt_id, __u32 idev_id)
43	49	{
44	50	struct iommu_test_cmd cmd = {
45	51	.size = sizeof(cmd),
@@ -57,14 +63,84 @@ static int _test_cmd_mock_domain(int fd, unsigned int ioas_id, __u32 *stdev_id, · 11 unmodified lines
57	63	assert(cmd.id != 0);
58	64	if (hwpt_id)
59	65	*hwpt_id = cmd.mock_domain.out_hwpt_id;
	66	+ if (idev_id)
	67	+ *idev_id = cmd.mock_domain.out_idev_id;
60	68	return 0;
61	69	}
62		−#define test_cmd_mock_domain(ioas_id, stdev_id, hwpt_id) \
63		− ASSERT_EQ(0, \
64		− _test_cmd_mock_domain(self->fd, ioas_id, stdev_id, hwpt_id))
	70	+#define test_cmd_mock_domain(ioas_id, stdev_id, hwpt_id, idev_id) \
	71	+ ASSERT_EQ(0, _test_cmd_mock_domain(self->fd, ioas_id, stdev_id, \
	72	+ hwpt_id, idev_id))
65	73	#define test_err_mock_domain(_errno, ioas_id, stdev_id, hwpt_id) \
66	74	EXPECT_ERRNO(_errno, _test_cmd_mock_domain(self->fd, ioas_id, \
67		− stdev_id, hwpt_id))
	75	+ stdev_id, hwpt_id, NULL))
	76	+
	77	+static int _test_cmd_mock_domain_replace(int fd, __u32 stdev_id, __u32 pt_id,
	78	+ __u32 *hwpt_id)
	79	+{
	80	+ struct iommu_test_cmd cmd = {
	81	+ .size = sizeof(cmd),
	82	+ .op = IOMMU_TEST_OP_MOCK_DOMAIN_REPLACE,
	83	+ .id = stdev_id,
	84	+ .mock_domain_replace = {
	85	+ .pt_id = pt_id,
	86	+ },
	87	+ };
	88	+ int ret;
	89	+
	90	+ ret = ioctl(fd, IOMMU_TEST_CMD, &cmd);
	91	+ if (ret)
	92	+ return ret;
	93	+ if (hwpt_id)
	94	+ *hwpt_id = cmd.mock_domain_replace.pt_id;
	95	+ return 0;
	96	+}
	97	+
	98	+#define test_cmd_mock_domain_replace(stdev_id, pt_id) \
	99	+ ASSERT_EQ(0, _test_cmd_mock_domain_replace(self->fd, stdev_id, pt_id, \
	100	+ NULL))
	101	+#define test_err_mock_domain_replace(_errno, stdev_id, pt_id) \
	102	+ EXPECT_ERRNO(_errno, _test_cmd_mock_domain_replace(self->fd, stdev_id, \
	103	+ pt_id, NULL))
	104	+
	105	+static int _test_cmd_hwpt_alloc(int fd, __u32 device_id, __u32 pt_id,
	106	+ __u32 *hwpt_id)
	107	+{
	108	+ struct iommu_hwpt_alloc cmd = {
	109	+ .size = sizeof(cmd),
	110	+ .dev_id = device_id,
	111	+ .pt_id = pt_id,
	112	+ };
	113	+ int ret;
	114	+
	115	+ ret = ioctl(fd, IOMMU_HWPT_ALLOC, &cmd);
	116	+ if (ret)
	117	+ return ret;
	118	+ if (hwpt_id)
	119	+ *hwpt_id = cmd.out_hwpt_id;
	120	+ return 0;
	121	+}
	122	+
	123	+#define test_cmd_hwpt_alloc(device_id, pt_id, hwpt_id) \
	124	+ ASSERT_EQ(0, _test_cmd_hwpt_alloc(self->fd, device_id, pt_id, hwpt_id))
	125	+
	126	+static int _test_cmd_access_replace_ioas(int fd, __u32 access_id,
	127	+ unsigned int ioas_id)
	128	+{
	129	+ struct iommu_test_cmd cmd = {
	130	+ .size = sizeof(cmd),
	131	+ .op = IOMMU_TEST_OP_ACCESS_REPLACE_IOAS,
	132	+ .id = access_id,
	133	+ .access_replace_ioas = { .ioas_id = ioas_id },
	134	+ };
	135	+ int ret;
	136	+
	137	+ ret = ioctl(fd, IOMMU_TEST_CMD, &cmd);
	138	+ if (ret)
	139	+ return ret;
	140	+ return 0;
	141	+}
	142	+#define test_cmd_access_replace_ioas(access_id, ioas_id) \
	143	+ ASSERT_EQ(0, _test_cmd_access_replace_ioas(self->fd, access_id, ioas_id))
68	144
69	145	static int _test_cmd_create_access(int fd, unsigned int ioas_id,
70	146	__u32 *access_id, unsigned int flags)
@@ -276,3 +352,61 @@ static void teardown_iommufd(int fd, struct __test_metadata *_metadata) · 205 unmodified lines
276	352	})
277	353
278	354	#endif
	355	+
	356	+/* @data can be NULL */
	357	+static int _test_cmd_get_hw_info(int fd, __u32 device_id,
	358	+ void *data, size_t data_len)
	359	+{
	360	+ struct iommu_test_hw_info info = (struct iommu_test_hw_info )data;
	361	+ struct iommu_hw_info cmd = {
	362	+ .size = sizeof(cmd),
	363	+ .dev_id = device_id,
	364	+ .data_len = data_len,
	365	+ .data_uptr = (uint64_t)data,
	366	+ };
	367	+ int ret;
	368	+
	369	+ ret = ioctl(fd, IOMMU_GET_HW_INFO, &cmd);
	370	+ if (ret)
	371	+ return ret;
	372	+
	373	+ assert(cmd.out_data_type == IOMMU_HW_INFO_TYPE_SELFTEST);
	374	+
	375	+ /*
	376	+ * The struct iommu_test_hw_info should be the one defined
	377	+ * by the current kernel.
	378	+ */
	379	+ assert(cmd.data_len == sizeof(struct iommu_test_hw_info));
	380	+
	381	+ /*
	382	+ * Trailing bytes should be 0 if user buffer is larger than
	383	+ * the data that kernel reports.
	384	+ */
	385	+ if (data_len > cmd.data_len) {
	386	+ char ptr = (char )(data + cmd.data_len);
	387	+ int idx = 0;
	388	+
	389	+ while (idx < data_len - cmd.data_len) {
	390	+ assert(!*(ptr + idx));
	391	+ idx++;
	392	+ }
	393	+ }
	394	+
	395	+ if (info) {
	396	+ if (data_len >= offsetofend(struct iommu_test_hw_info, test_reg))
	397	+ assert(info->test_reg == IOMMU_HW_INFO_SELFTEST_REGVAL);
	398	+ if (data_len >= offsetofend(struct iommu_test_hw_info, flags))
	399	+ assert(!info->flags);
	400	+ }
	401	+
	402	+ return 0;
	403	+}
	404	+
	405	+#define test_cmd_get_hw_info(device_id, data, data_len) \
	406	+ ASSERT_EQ(0, _test_cmd_get_hw_info(self->fd, device_id, \
	407	+ data, data_len))
	408	+
	409	+#define test_err_get_hw_info(_errno, device_id, data, data_len) \
	410	+ EXPECT_ERRNO(_errno, \
	411	+ _test_cmd_get_hw_info(self->fd, device_id, \
	412	+ data, data_len))