lael-labs/linux

drivers/iommu/intel/iommu.c+19−0

@@ -22,6 +22,7 @@ · 21 unmodified lines
22	#include <linux/spinlock.h>	22	#include <linux/spinlock.h>
23	#include <linux/syscore_ops.h>	23	#include <linux/syscore_ops.h>
24	#include <linux/tboot.h>	24	#include <linux/tboot.h>
		25	#include <uapi/linux/iommufd.h>
25		26
26	#include "iommu.h"	27	#include "iommu.h"
27	#include "../dma-iommu.h"	28	#include "../dma-iommu.h"
@@ -4732,8 +4733,26 @@ static void intel_iommu_remove_dev_pasid(struct device *dev, ioasid_t pasid) · 4704 unmodified lines
4732	intel_pasid_tear_down_entry(iommu, dev, pasid, false);	4733	intel_pasid_tear_down_entry(iommu, dev, pasid, false);
4733	}	4734	}
4734		4735
		4736	static void intel_iommu_hw_info(struct device dev, u32 length, u32 type)
		4737	{
		4738	struct device_domain_info *info = dev_iommu_priv_get(dev);
		4739	struct intel_iommu *iommu = info->iommu;
		4740	struct iommu_hw_info_vtd *vtd;
		4741
		4742	vtd = kzalloc(sizeof(*vtd), GFP_KERNEL);
		4743	if (!vtd)
		4744	return ERR_PTR(-ENOMEM);
		4745
		4746	vtd->cap_reg = iommu->cap;
		4747	vtd->ecap_reg = iommu->ecap;
		4748	length = sizeof(vtd);
		4749	*type = IOMMU_HW_INFO_TYPE_INTEL_VTD;
		4750	return vtd;
		4751	}
		4752
4735	const struct iommu_ops intel_iommu_ops = {	4753	const struct iommu_ops intel_iommu_ops = {
4736	.capable = intel_iommu_capable,	4754	.capable = intel_iommu_capable,
		4755	.hw_info = intel_iommu_hw_info,
4737	.domain_alloc = intel_iommu_domain_alloc,	4756	.domain_alloc = intel_iommu_domain_alloc,
4738	.probe_device = intel_iommu_probe_device,	4757	.probe_device = intel_iommu_probe_device,
4739	.probe_finalize = intel_iommu_probe_finalize,	4758	.probe_finalize = intel_iommu_probe_finalize,

drivers/iommu/iommu-priv.h+30−0

@@ -0,0 +1,30 @@
	1	/* SPDX-License-Identifier: GPL-2.0-only */
	2	/* Copyright (c) 2023, NVIDIA CORPORATION & AFFILIATES.
	3	*/
	4	#ifndef __LINUX_IOMMU_PRIV_H
	5	#define __LINUX_IOMMU_PRIV_H
	6
	7	#include <linux/iommu.h>
	8
	9	static inline const struct iommu_ops dev_iommu_ops(struct device dev)
	10	{
	11	/*
	12	* Assume that valid ops must be installed if iommu_probe_device()
	13	* has succeeded. The device ops are essentially for internal use
	14	* within the IOMMU subsystem itself, so we should be able to trust
	15	* ourselves not to misuse the helper.
	16	*/
	17	return dev->iommu->iommu_dev->ops;
	18	}
	19
	20	int iommu_group_replace_domain(struct iommu_group *group,
	21	struct iommu_domain *new_domain);
	22
	23	int iommu_device_register_bus(struct iommu_device *iommu,
	24	const struct iommu_ops ops, struct bus_type bus,
	25	struct notifier_block *nb);
	26	void iommu_device_unregister_bus(struct iommu_device *iommu,
	27	struct bus_type *bus,
	28	struct notifier_block *nb);
	29
	30	#endif /* __LINUX_IOMMU_PRIV_H */

drivers/iommu/iommu.c+80−1

@@ -34,8 +34,10 @@ · 33 unmodified lines
34	#include <linux/msi.h>	34	#include <linux/msi.h>
35		35
36	#include "dma-iommu.h"	36	#include "dma-iommu.h"
		37	#include "iommu-priv.h"
37		38
38	#include "iommu-sva.h"	39	#include "iommu-sva.h"
		40	#include "iommu-priv.h"
39		41
40	static struct kset *iommu_group_kset;	42	static struct kset *iommu_group_kset;
41	static DEFINE_IDA(iommu_group_ida);	43	static DEFINE_IDA(iommu_group_ida);
@@ -287,6 +289,48 @@ void iommu_device_unregister(struct iommu_device *iommu) · 245 unmodified lines
287	}	289	}
288	EXPORT_SYMBOL_GPL(iommu_device_unregister);	290	EXPORT_SYMBOL_GPL(iommu_device_unregister);
289		291
		292	#if IS_ENABLED(CONFIG_IOMMUFD_TEST)
		293	void iommu_device_unregister_bus(struct iommu_device *iommu,
		294	struct bus_type *bus,
		295	struct notifier_block *nb)
		296	{
		297	bus_unregister_notifier(bus, nb);
		298	iommu_device_unregister(iommu);
		299	}
		300	EXPORT_SYMBOL_GPL(iommu_device_unregister_bus);
		301
		302	/*
		303	* Register an iommu driver against a single bus. This is only used by iommufd
		304	* selftest to create a mock iommu driver. The caller must provide
		305	* some memory to hold a notifier_block.
		306	*/
		307	int iommu_device_register_bus(struct iommu_device *iommu,
		308	const struct iommu_ops ops, struct bus_type bus,
		309	struct notifier_block *nb)
		310	{
		311	int err;
		312
		313	iommu->ops = ops;
		314	nb->notifier_call = iommu_bus_notifier;
		315	err = bus_register_notifier(bus, nb);
		316	if (err)
		317	return err;
		318
		319	spin_lock(&iommu_device_lock);
		320	list_add_tail(&iommu->list, &iommu_device_list);
		321	spin_unlock(&iommu_device_lock);
		322
		323	bus->iommu_ops = ops;
		324	err = bus_iommu_probe(bus);
		325	if (err) {
		326	iommu_device_unregister_bus(iommu, bus, nb);
		327	return err;
		328	}
		329	return 0;
		330	}
		331	EXPORT_SYMBOL_GPL(iommu_device_register_bus);
		332	#endif
		333
290	static struct dev_iommu dev_iommu_get(struct device dev)	334	static struct dev_iommu dev_iommu_get(struct device dev)
291	{	335	{
292	struct dev_iommu *param = dev->iommu;	336	struct dev_iommu *param = dev->iommu;
@@ -2114,6 +2158,32 @@ int iommu_attach_group(struct iommu_domain domain, struct iommu_group group) · 1821 unmodified lines
2114	}	2158	}
2115	EXPORT_SYMBOL_GPL(iommu_attach_group);	2159	EXPORT_SYMBOL_GPL(iommu_attach_group);
2116		2160
		2161	/**
		2162	* iommu_group_replace_domain - replace the domain that a group is attached to
		2163	* @new_domain: new IOMMU domain to replace with
		2164	* @group: IOMMU group that will be attached to the new domain
		2165	*
		2166	* This API allows the group to switch domains without being forced to go to
		2167	* the blocking domain in-between.
		2168	*
		2169	* If the currently attached domain is a core domain (e.g. a default_domain),
		2170	* it will act just like the iommu_attach_group().
		2171	*/
		2172	int iommu_group_replace_domain(struct iommu_group *group,
		2173	struct iommu_domain *new_domain)
		2174	{
		2175	int ret;
		2176
		2177	if (!new_domain)
		2178	return -EINVAL;
		2179
		2180	mutex_lock(&group->mutex);
		2181	ret = __iommu_group_set_domain(group, new_domain);
		2182	mutex_unlock(&group->mutex);
		2183	return ret;
		2184	}
		2185	EXPORT_SYMBOL_NS_GPL(iommu_group_replace_domain, IOMMUFD_INTERNAL);
		2186
2117	static int __iommu_device_set_domain(struct iommu_group *group,	2187	static int __iommu_device_set_domain(struct iommu_group *group,
2118	struct device *dev,	2188	struct device *dev,
2119	struct iommu_domain *new_domain,	2189	struct iommu_domain *new_domain,
@@ -2642,6 +2712,14 @@ int iommu_set_pgtable_quirks(struct iommu_domain *domain, · 522 unmodified lines
2642	}	2712	}
2643	EXPORT_SYMBOL_GPL(iommu_set_pgtable_quirks);	2713	EXPORT_SYMBOL_GPL(iommu_set_pgtable_quirks);
2644		2714
		2715	/**
		2716	* iommu_get_resv_regions - get reserved regions
		2717	* @dev: device for which to get reserved regions
		2718	* @list: reserved region list for device
		2719	*
		2720	* This returns a list of reserved IOVA regions specific to this device.
		2721	* A domain user should not map IOVA in these ranges.
		2722	*/
2645	void iommu_get_resv_regions(struct device dev, struct list_head list)	2723	void iommu_get_resv_regions(struct device dev, struct list_head list)
2646	{	2724	{
2647	const struct iommu_ops *ops = dev_iommu_ops(dev);	2725	const struct iommu_ops *ops = dev_iommu_ops(dev);
@@ -2649,9 +2727,10 @@ void iommu_get_resv_regions(struct device dev, struct list_head list) · 1 unmodified line
2649	if (ops->get_resv_regions)	2727	if (ops->get_resv_regions)
2650	ops->get_resv_regions(dev, list);	2728	ops->get_resv_regions(dev, list);
2651	}	2729	}
		2730	EXPORT_SYMBOL_GPL(iommu_get_resv_regions);
2652		2731
2653	/**	2732	/**
2654	* iommu_put_resv_regions - release resered regions	2733	* iommu_put_resv_regions - release reserved regions
2655	* @dev: device for which to free reserved regions	2734	* @dev: device for which to free reserved regions
2656	* @list: reserved region list for device	2735	* @list: reserved region list for device
2657	*	2736	*

drivers/iommu/iommufd/device.c+552−179

@@ -4,6 +4,8 @@ · 3 unmodified lines
4	#include <linux/iommufd.h>	4	#include <linux/iommufd.h>
5	#include <linux/slab.h>	5	#include <linux/slab.h>
6	#include <linux/iommu.h>	6	#include <linux/iommu.h>
		7	#include <uapi/linux/iommufd.h>
		8	#include "../iommu-priv.h"
7		9
8	#include "io_pagetable.h"	10	#include "io_pagetable.h"
9	#include "iommufd_private.h"	11	#include "iommufd_private.h"
@@ -15,13 +17,127 @@ MODULE_PARM_DESC( · 5 unmodified lines
15	"Allow IOMMUFD to bind to devices even if the platform cannot isolate "	17	"Allow IOMMUFD to bind to devices even if the platform cannot isolate "
16	"the MSI interrupt window. Enabling this is a security weakness.");	18	"the MSI interrupt window. Enabling this is a security weakness.");
17		19
		20	static void iommufd_group_release(struct kref *kref)
		21	{
		22	struct iommufd_group *igroup =
		23	container_of(kref, struct iommufd_group, ref);
		24
		25	WARN_ON(igroup->hwpt \|\| !list_empty(&igroup->device_list));
		26
		27	xa_cmpxchg(&igroup->ictx->groups, iommu_group_id(igroup->group), igroup,
		28	NULL, GFP_KERNEL);
		29	iommu_group_put(igroup->group);
		30	mutex_destroy(&igroup->lock);
		31	kfree(igroup);
		32	}
		33
		34	static void iommufd_put_group(struct iommufd_group *group)
		35	{
		36	kref_put(&group->ref, iommufd_group_release);
		37	}
		38
		39	static bool iommufd_group_try_get(struct iommufd_group *igroup,
		40	struct iommu_group *group)
		41	{
		42	if (!igroup)
		43	return false;
		44	/*
		45	* group ID's cannot be re-used until the group is put back which does
		46	* not happen if we could get an igroup pointer under the xa_lock.
		47	*/
		48	if (WARN_ON(igroup->group != group))
		49	return false;
		50	return kref_get_unless_zero(&igroup->ref);
		51	}
		52
		53	/*
		54	* iommufd needs to store some more data for each iommu_group, we keep a
		55	* parallel xarray indexed by iommu_group id to hold this instead of putting it
		56	* in the core structure. To keep things simple the iommufd_group memory is
		57	* unique within the iommufd_ctx. This makes it easy to check there are no
		58	* memory leaks.
		59	*/
		60	static struct iommufd_group iommufd_get_group(struct iommufd_ctx ictx,
		61	struct device *dev)
		62	{
		63	struct iommufd_group *new_igroup;
		64	struct iommufd_group *cur_igroup;
		65	struct iommufd_group *igroup;
		66	struct iommu_group *group;
		67	unsigned int id;
		68
		69	group = iommu_group_get(dev);
		70	if (!group)
		71	return ERR_PTR(-ENODEV);
		72
		73	id = iommu_group_id(group);
		74
		75	xa_lock(&ictx->groups);
		76	igroup = xa_load(&ictx->groups, id);
		77	if (iommufd_group_try_get(igroup, group)) {
		78	xa_unlock(&ictx->groups);
		79	iommu_group_put(group);
		80	return igroup;
		81	}
		82	xa_unlock(&ictx->groups);
		83
		84	new_igroup = kzalloc(sizeof(*new_igroup), GFP_KERNEL);
		85	if (!new_igroup) {
		86	iommu_group_put(group);
		87	return ERR_PTR(-ENOMEM);
		88	}
		89
		90	kref_init(&new_igroup->ref);
		91	mutex_init(&new_igroup->lock);
		92	INIT_LIST_HEAD(&new_igroup->device_list);
		93	new_igroup->sw_msi_start = PHYS_ADDR_MAX;
		94	/* group reference moves into new_igroup */
		95	new_igroup->group = group;
		96
		97	/*
		98	* The ictx is not additionally refcounted here becase all objects using
		99	* an igroup must put it before their destroy completes.
		100	*/
		101	new_igroup->ictx = ictx;
		102
		103	/*
		104	* We dropped the lock so igroup is invalid. NULL is a safe and likely
		105	* value to assume for the xa_cmpxchg algorithm.
		106	*/
		107	cur_igroup = NULL;
		108	xa_lock(&ictx->groups);
		109	while (true) {
		110	igroup = __xa_cmpxchg(&ictx->groups, id, cur_igroup, new_igroup,
		111	GFP_KERNEL);
		112	if (xa_is_err(igroup)) {
		113	xa_unlock(&ictx->groups);
		114	iommufd_put_group(new_igroup);
		115	return ERR_PTR(xa_err(igroup));
		116	}
		117
		118	/* new_group was successfully installed */
		119	if (cur_igroup == igroup) {
		120	xa_unlock(&ictx->groups);
		121	return new_igroup;
		122	}
		123
		124	/* Check again if the current group is any good */
		125	if (iommufd_group_try_get(igroup, group)) {
		126	xa_unlock(&ictx->groups);
		127	iommufd_put_group(new_igroup);
		128	return igroup;
		129	}
		130	cur_igroup = igroup;
		131	}
		132	}
		133
18	void iommufd_device_destroy(struct iommufd_object *obj)	134	void iommufd_device_destroy(struct iommufd_object *obj)
19	{	135	{
20	struct iommufd_device *idev =	136	struct iommufd_device *idev =
21	container_of(obj, struct iommufd_device, obj);	137	container_of(obj, struct iommufd_device, obj);
22		138
23	iommu_device_release_dma_owner(idev->dev);	139	iommu_device_release_dma_owner(idev->dev);
24	iommu_group_put(idev->group);	140	iommufd_put_group(idev->igroup);
25	if (!iommufd_selftest_is_mock_dev(idev->dev))	141	if (!iommufd_selftest_is_mock_dev(idev->dev))
26	iommufd_ctx_put(idev->ictx);	142	iommufd_ctx_put(idev->ictx);
27	}	143	}
@@ -46,7 +162,7 @@ struct iommufd_device iommufd_device_bind(struct iommufd_ctx ictx, · 18 unmodified lines
46	struct device dev, u32 id)	162	struct device dev, u32 id)
47	{	163	{
48	struct iommufd_device *idev;	164	struct iommufd_device *idev;
49	struct iommu_group *group;	165	struct iommufd_group *igroup;
50	int rc;	166	int rc;
51		167
52	/*	168	/*
@@ -56,9 +172,29 @@ struct iommufd_device iommufd_device_bind(struct iommufd_ctx ictx, · 3 unmodified lines
56	if (!device_iommu_capable(dev, IOMMU_CAP_CACHE_COHERENCY))	172	if (!device_iommu_capable(dev, IOMMU_CAP_CACHE_COHERENCY))
57	return ERR_PTR(-EINVAL);	173	return ERR_PTR(-EINVAL);
58		174
59	group = iommu_group_get(dev);	175	igroup = iommufd_get_group(ictx, dev);
60	if (!group)	176	if (IS_ERR(igroup))
61	return ERR_PTR(-ENODEV);	177	return ERR_CAST(igroup);
		178
		179	/*
		180	* For historical compat with VFIO the insecure interrupt path is
		181	* allowed if the module parameter is set. Secure/Isolated means that a
		182	* MemWr operation from the device (eg a simple DMA) cannot trigger an
		183	* interrupt outside this iommufd context.
		184	*/
		185	if (!iommufd_selftest_is_mock_dev(dev) &&
		186	!iommu_group_has_isolated_msi(igroup->group)) {
		187	if (!allow_unsafe_interrupts) {
		188	rc = -EPERM;
		189	goto out_group_put;
		190	}
		191
		192	dev_warn(
		193	dev,
		194	"MSI interrupts are not secure, they cannot be isolated by the platform. "
		195	"Check that platform features like interrupt remapping are enabled. "
		196	"Use the \"allow_unsafe_interrupts\" module parameter to override\n");
		197	}
62		198
63	rc = iommu_device_claim_dma_owner(dev, ictx);	199	rc = iommu_device_claim_dma_owner(dev, ictx);
64	if (rc)	200	if (rc)
@@ -77,8 +213,8 @@ struct iommufd_device iommufd_device_bind(struct iommufd_ctx ictx, · 12 unmodified lines
77	device_iommu_capable(dev, IOMMU_CAP_ENFORCE_CACHE_COHERENCY);	213	device_iommu_capable(dev, IOMMU_CAP_ENFORCE_CACHE_COHERENCY);
78	/* The calling driver is a user until iommufd_device_unbind() */	214	/* The calling driver is a user until iommufd_device_unbind() */
79	refcount_inc(&idev->obj.users);	215	refcount_inc(&idev->obj.users);
80	/* group refcount moves into iommufd_device */	216	/* igroup refcount moves into iommufd_device */
81	idev->group = group;	217	idev->igroup = igroup;
82		218
83	/*	219	/*
84	* If the caller fails after this success it must call	220	* If the caller fails after this success it must call
@@ -93,7 +229,7 @@ struct iommufd_device iommufd_device_bind(struct iommufd_ctx ictx, · 8 unmodified lines
93	out_release_owner:	229	out_release_owner:
94	iommu_device_release_dma_owner(dev);	230	iommu_device_release_dma_owner(dev);
95	out_group_put:	231	out_group_put:
96	iommu_group_put(group);	232	iommufd_put_group(igroup);
97	return ERR_PTR(rc);	233	return ERR_PTR(rc);
98	}	234	}
99	EXPORT_SYMBOL_NS_GPL(iommufd_device_bind, IOMMUFD);	235	EXPORT_SYMBOL_NS_GPL(iommufd_device_bind, IOMMUFD);
@@ -118,7 +254,8 @@ bool iommufd_ctx_has_group(struct iommufd_ctx ictx, struct iommu_group group) · 18 unmodified lines
118	xa_lock(&ictx->objects);	254	xa_lock(&ictx->objects);
119	xa_for_each(&ictx->objects, index, obj) {	255	xa_for_each(&ictx->objects, index, obj) {
120	if (obj->type == IOMMUFD_OBJ_DEVICE &&	256	if (obj->type == IOMMUFD_OBJ_DEVICE &&
121	container_of(obj, struct iommufd_device, obj)->group == group) {	257	container_of(obj, struct iommufd_device, obj)
		258	->igroup->group == group) {
122	xa_unlock(&ictx->objects);	259	xa_unlock(&ictx->objects);
123	return true;	260	return true;
124	}	261	}
@@ -155,10 +292,10 @@ u32 iommufd_device_to_id(struct iommufd_device *idev) · 30 unmodified lines
155	}	292	}
156	EXPORT_SYMBOL_NS_GPL(iommufd_device_to_id, IOMMUFD);	293	EXPORT_SYMBOL_NS_GPL(iommufd_device_to_id, IOMMUFD);
157		294
158	static int iommufd_device_setup_msi(struct iommufd_device *idev,	295	static int iommufd_group_setup_msi(struct iommufd_group *igroup,
159	struct iommufd_hw_pagetable *hwpt,	296	struct iommufd_hw_pagetable *hwpt)
160	phys_addr_t sw_msi_start)
161	{	297	{
		298	phys_addr_t sw_msi_start = igroup->sw_msi_start;
162	int rc;	299	int rc;
163		300
164	/*	301	/*
@@ -185,128 +322,192 @@ static int iommufd_device_setup_msi(struct iommufd_device *idev, · 20 unmodified lines
185	*/	322	*/
186	hwpt->msi_cookie = true;	323	hwpt->msi_cookie = true;
187	}	324	}
188
189	/*
190	* For historical compat with VFIO the insecure interrupt path is
191	* allowed if the module parameter is set. Insecure means that a MemWr
192	* operation from the device (eg a simple DMA) cannot trigger an
193	* interrupt outside this iommufd context.
194	*/
195	if (!iommufd_selftest_is_mock_dev(idev->dev) &&
196	!iommu_group_has_isolated_msi(idev->group)) {
197	if (!allow_unsafe_interrupts)
198	return -EPERM;
199
200	dev_warn(
201	idev->dev,
202	"MSI interrupts are not secure, they cannot be isolated by the platform. "
203	"Check that platform features like interrupt remapping are enabled. "
204	"Use the \"allow_unsafe_interrupts\" module parameter to override\n");
205	}
206	return 0;	325	return 0;
207	}	326	}
208		327
209	static bool iommufd_hw_pagetable_has_group(struct iommufd_hw_pagetable *hwpt,
210	struct iommu_group *group)
211	{
212	struct iommufd_device *cur_dev;
213
214	lockdep_assert_held(&hwpt->devices_lock);
215
216	list_for_each_entry(cur_dev, &hwpt->devices, devices_item)
217	if (cur_dev->group == group)
218	return true;
219	return false;
220	}
221
222	int iommufd_hw_pagetable_attach(struct iommufd_hw_pagetable *hwpt,	328	int iommufd_hw_pagetable_attach(struct iommufd_hw_pagetable *hwpt,
223	struct iommufd_device *idev)	329	struct iommufd_device *idev)
224	{	330	{
225	phys_addr_t sw_msi_start = PHYS_ADDR_MAX;
226	int rc;	331	int rc;
227		332
228	lockdep_assert_held(&hwpt->devices_lock);	333	mutex_lock(&idev->igroup->lock);
229
230	if (WARN_ON(idev->hwpt))
231	return -EINVAL;
232		334
233	/*	335	if (idev->igroup->hwpt != NULL && idev->igroup->hwpt != hwpt) {
234	* Try to upgrade the domain we have, it is an iommu driver bug to	336	rc = -EINVAL;
235	* report IOMMU_CAP_ENFORCE_CACHE_COHERENCY but fail	337	goto err_unlock;
236	* enforce_cache_coherency when there are no devices attached to the
237	* domain.
238	*/
239	if (idev->enforce_cache_coherency && !hwpt->enforce_cache_coherency) {
240	if (hwpt->domain->ops->enforce_cache_coherency)
241	hwpt->enforce_cache_coherency =
242	hwpt->domain->ops->enforce_cache_coherency(
243	hwpt->domain);
244	if (!hwpt->enforce_cache_coherency) {
245	WARN_ON(list_empty(&hwpt->devices));
246	return -EINVAL;
247	}
248	}	338	}
249		339
250	rc = iopt_table_enforce_group_resv_regions(&hwpt->ioas->iopt, idev->dev,	340	/* Try to upgrade the domain we have */
251	idev->group, &sw_msi_start);	341	if (idev->enforce_cache_coherency) {
252	if (rc)	342	rc = iommufd_hw_pagetable_enforce_cc(hwpt);
253	return rc;	343	if (rc)
		344	goto err_unlock;
		345	}
254		346
255	rc = iommufd_device_setup_msi(idev, hwpt, sw_msi_start);	347	rc = iopt_table_enforce_dev_resv_regions(&hwpt->ioas->iopt, idev->dev,
		348	&idev->igroup->sw_msi_start);
256	if (rc)	349	if (rc)
257	goto err_unresv;	350	goto err_unlock;
258		351
259	/*	352	/*
260	* FIXME: Hack around missing a device-centric iommu api, only attach to	353	* Only attach to the group once for the first device that is in the
261	* the group once for the first device that is in the group.	354	* group. All the other devices will follow this attachment. The user
		355	* should attach every device individually to the hwpt as the per-device
		356	* reserved regions are only updated during individual device
		357	* attachment.
262	*/	358	*/
263	if (!iommufd_hw_pagetable_has_group(hwpt, idev->group)) {	359	if (list_empty(&idev->igroup->device_list)) {
264	rc = iommu_attach_group(hwpt->domain, idev->group);	360	rc = iommufd_group_setup_msi(idev->igroup, hwpt);
265	if (rc)	361	if (rc)
266	goto err_unresv;	362	goto err_unresv;
		363
		364	rc = iommu_attach_group(hwpt->domain, idev->igroup->group);
		365	if (rc)
		366	goto err_unresv;
		367	idev->igroup->hwpt = hwpt;
267	}	368	}
		369	refcount_inc(&hwpt->obj.users);
		370	list_add_tail(&idev->group_item, &idev->igroup->device_list);
		371	mutex_unlock(&idev->igroup->lock);
268	return 0;	372	return 0;
269	err_unresv:	373	err_unresv:
270	iopt_remove_reserved_iova(&hwpt->ioas->iopt, idev->dev);	374	iopt_remove_reserved_iova(&hwpt->ioas->iopt, idev->dev);
		375	err_unlock:
		376	mutex_unlock(&idev->igroup->lock);
271	return rc;	377	return rc;
272	}	378	}
273		379
274	void iommufd_hw_pagetable_detach(struct iommufd_hw_pagetable *hwpt,	380	struct iommufd_hw_pagetable *
275	struct iommufd_device *idev)	381	iommufd_hw_pagetable_detach(struct iommufd_device *idev)
276	{	382	{
277	if (!iommufd_hw_pagetable_has_group(hwpt, idev->group))	383	struct iommufd_hw_pagetable *hwpt = idev->igroup->hwpt;
278	iommu_detach_group(hwpt->domain, idev->group);	384
		385	mutex_lock(&idev->igroup->lock);
		386	list_del(&idev->group_item);
		387	if (list_empty(&idev->igroup->device_list)) {
		388	iommu_detach_group(hwpt->domain, idev->igroup->group);
		389	idev->igroup->hwpt = NULL;
		390	}
279	iopt_remove_reserved_iova(&hwpt->ioas->iopt, idev->dev);	391	iopt_remove_reserved_iova(&hwpt->ioas->iopt, idev->dev);
		392	mutex_unlock(&idev->igroup->lock);
		393
		394	/* Caller must destroy hwpt */
		395	return hwpt;
280	}	396	}
281		397
282	static int iommufd_device_do_attach(struct iommufd_device *idev,	398	static struct iommufd_hw_pagetable *
283	struct iommufd_hw_pagetable *hwpt)	399	iommufd_device_do_attach(struct iommufd_device *idev,
		400	struct iommufd_hw_pagetable *hwpt)
284	{	401	{
285	int rc;	402	int rc;
286		403
287	mutex_lock(&hwpt->devices_lock);
288	rc = iommufd_hw_pagetable_attach(hwpt, idev);	404	rc = iommufd_hw_pagetable_attach(hwpt, idev);
289	if (rc)	405	if (rc)
290	goto out_unlock;	406	return ERR_PTR(rc);
		407	return NULL;
		408	}
291		409
292	idev->hwpt = hwpt;	410	static struct iommufd_hw_pagetable *
293	refcount_inc(&hwpt->obj.users);	411	iommufd_device_do_replace(struct iommufd_device *idev,
294	list_add(&idev->devices_item, &hwpt->devices);	412	struct iommufd_hw_pagetable *hwpt)
295	out_unlock:	413	{
296	mutex_unlock(&hwpt->devices_lock);	414	struct iommufd_group *igroup = idev->igroup;
297	return rc;	415	struct iommufd_hw_pagetable *old_hwpt;
		416	unsigned int num_devices = 0;
		417	struct iommufd_device *cur;
		418	int rc;
		419
		420	mutex_lock(&idev->igroup->lock);
		421
		422	if (igroup->hwpt == NULL) {
		423	rc = -EINVAL;
		424	goto err_unlock;
		425	}
		426
		427	if (hwpt == igroup->hwpt) {
		428	mutex_unlock(&idev->igroup->lock);
		429	return NULL;
		430	}
		431
		432	/* Try to upgrade the domain we have */
		433	list_for_each_entry(cur, &igroup->device_list, group_item) {
		434	num_devices++;
		435	if (cur->enforce_cache_coherency) {
		436	rc = iommufd_hw_pagetable_enforce_cc(hwpt);
		437	if (rc)
		438	goto err_unlock;
		439	}
		440	}
		441
		442	old_hwpt = igroup->hwpt;
		443	if (hwpt->ioas != old_hwpt->ioas) {
		444	list_for_each_entry(cur, &igroup->device_list, group_item) {
		445	rc = iopt_table_enforce_dev_resv_regions(
		446	&hwpt->ioas->iopt, cur->dev, NULL);
		447	if (rc)
		448	goto err_unresv;
		449	}
		450	}
		451
		452	rc = iommufd_group_setup_msi(idev->igroup, hwpt);
		453	if (rc)
		454	goto err_unresv;
		455
		456	rc = iommu_group_replace_domain(igroup->group, hwpt->domain);
		457	if (rc)
		458	goto err_unresv;
		459
		460	if (hwpt->ioas != old_hwpt->ioas) {
		461	list_for_each_entry(cur, &igroup->device_list, group_item)
		462	iopt_remove_reserved_iova(&old_hwpt->ioas->iopt,
		463	cur->dev);
		464	}
		465
		466	igroup->hwpt = hwpt;
		467
		468	/*
		469	* Move the refcounts held by the device_list to the new hwpt. Retain a
		470	* refcount for this thread as the caller will free it.
		471	*/
		472	refcount_add(num_devices, &hwpt->obj.users);
		473	if (num_devices > 1)
		474	WARN_ON(refcount_sub_and_test(num_devices - 1,
		475	&old_hwpt->obj.users));
		476	mutex_unlock(&idev->igroup->lock);
		477
		478	/* Caller must destroy old_hwpt */
		479	return old_hwpt;
		480	err_unresv:
		481	list_for_each_entry(cur, &igroup->device_list, group_item)
		482	iopt_remove_reserved_iova(&hwpt->ioas->iopt, cur->dev);
		483	err_unlock:
		484	mutex_unlock(&idev->igroup->lock);
		485	return ERR_PTR(rc);
298	}	486	}
299		487
		488	typedef struct iommufd_hw_pagetable (attach_fn)(
		489	struct iommufd_device idev, struct iommufd_hw_pagetable hwpt);
		490
300	/*	491	/*
301	* When automatically managing the domains we search for a compatible domain in	492	* When automatically managing the domains we search for a compatible domain in
302	* the iopt and if one is found use it, otherwise create a new domain.	493	* the iopt and if one is found use it, otherwise create a new domain.
303	* Automatic domain selection will never pick a manually created domain.	494	* Automatic domain selection will never pick a manually created domain.
304	*/	495	*/
305	static int iommufd_device_auto_get_domain(struct iommufd_device *idev,	496	static struct iommufd_hw_pagetable *
306	struct iommufd_ioas *ioas)	497	iommufd_device_auto_get_domain(struct iommufd_device *idev,
		498	struct iommufd_ioas ioas, u32 pt_id,
		499	attach_fn do_attach)
307	{	500	{
		501	/*
		502	* iommufd_hw_pagetable_attach() is called by
		503	* iommufd_hw_pagetable_alloc() in immediate attachment mode, same as
		504	* iommufd_device_do_attach(). So if we are in this mode then we prefer
		505	* to use the immediate_attach path as it supports drivers that can't
		506	* directly allocate a domain.
		507	*/
		508	bool immediate_attach = do_attach == iommufd_device_do_attach;
		509	struct iommufd_hw_pagetable *destroy_hwpt;
308	struct iommufd_hw_pagetable *hwpt;	510	struct iommufd_hw_pagetable *hwpt;
309	int rc;
310		511
311	/*	512	/*
312	* There is no differentiation when domains are allocated, so any domain	513	* There is no differentiation when domains are allocated, so any domain
@@ -320,50 +521,58 @@ static int iommufd_device_auto_get_domain(struct iommufd_device *idev, · 7 unmodified lines
320		521
321	if (!iommufd_lock_obj(&hwpt->obj))	522	if (!iommufd_lock_obj(&hwpt->obj))
322	continue;	523	continue;
323	rc = iommufd_device_do_attach(idev, hwpt);	524	destroy_hwpt = (*do_attach)(idev, hwpt);
		525	if (IS_ERR(destroy_hwpt)) {
		526	iommufd_put_object(&hwpt->obj);
		527	/*
		528	* -EINVAL means the domain is incompatible with the
		529	* device. Other error codes should propagate to
		530	* userspace as failure. Success means the domain is
		531	* attached.
		532	*/
		533	if (PTR_ERR(destroy_hwpt) == -EINVAL)
		534	continue;
		535	goto out_unlock;
		536	}
		537	*pt_id = hwpt->obj.id;
324	iommufd_put_object(&hwpt->obj);	538	iommufd_put_object(&hwpt->obj);
325
326	/*
327	* -EINVAL means the domain is incompatible with the device.
328	* Other error codes should propagate to userspace as failure.
329	* Success means the domain is attached.
330	*/
331	if (rc == -EINVAL)
332	continue;
333	goto out_unlock;	539	goto out_unlock;
334	}	540	}
335		541
336	hwpt = iommufd_hw_pagetable_alloc(idev->ictx, ioas, idev, true);	542	hwpt = iommufd_hw_pagetable_alloc(idev->ictx, ioas, idev,
		543	immediate_attach);
337	if (IS_ERR(hwpt)) {	544	if (IS_ERR(hwpt)) {
338	rc = PTR_ERR(hwpt);	545	destroy_hwpt = ERR_CAST(hwpt);
339	goto out_unlock;	546	goto out_unlock;
340	}	547	}
		548
		549	if (!immediate_attach) {
		550	destroy_hwpt = (*do_attach)(idev, hwpt);
		551	if (IS_ERR(destroy_hwpt))
		552	goto out_abort;
		553	} else {
		554	destroy_hwpt = NULL;
		555	}
		556
341	hwpt->auto_domain = true;	557	hwpt->auto_domain = true;
		558	*pt_id = hwpt->obj.id;
342		559
343	mutex_unlock(&ioas->mutex);
344	iommufd_object_finalize(idev->ictx, &hwpt->obj);	560	iommufd_object_finalize(idev->ictx, &hwpt->obj);
345	return 0;	561	mutex_unlock(&ioas->mutex);
		562	return destroy_hwpt;
		563
		564	out_abort:
		565	iommufd_object_abort_and_destroy(idev->ictx, &hwpt->obj);
346	out_unlock:	566	out_unlock:
347	mutex_unlock(&ioas->mutex);	567	mutex_unlock(&ioas->mutex);
348	return rc;	568	return destroy_hwpt;
349	}	569	}
350		570
351	/**	571	static int iommufd_device_change_pt(struct iommufd_device idev, u32 pt_id,
352	* iommufd_device_attach - Connect a device from an iommu_domain	572	attach_fn do_attach)
353	* @idev: device to attach
354	* @pt_id: Input a IOMMUFD_OBJ_IOAS, or IOMMUFD_OBJ_HW_PAGETABLE
355	* Output the IOMMUFD_OBJ_HW_PAGETABLE ID
356	*
357	* This connects the device to an iommu_domain, either automatically or manually
358	* selected. Once this completes the device could do DMA.
359	*
360	* The caller should return the resulting pt_id back to userspace.
361	* This function is undone by calling iommufd_device_detach().
362	*/
363	int iommufd_device_attach(struct iommufd_device idev, u32 pt_id)
364	{	573	{
		574	struct iommufd_hw_pagetable *destroy_hwpt;
365	struct iommufd_object *pt_obj;	575	struct iommufd_object *pt_obj;
366	int rc;
367		576
368	pt_obj = iommufd_get_object(idev->ictx, *pt_id, IOMMUFD_OBJ_ANY);	577	pt_obj = iommufd_get_object(idev->ictx, *pt_id, IOMMUFD_OBJ_ANY);
369	if (IS_ERR(pt_obj))	578	if (IS_ERR(pt_obj))
@@ -374,8 +583,8 @@ int iommufd_device_attach(struct iommufd_device idev, u32 pt_id) · 4 unmodified lines
374	struct iommufd_hw_pagetable *hwpt =	583	struct iommufd_hw_pagetable *hwpt =
375	container_of(pt_obj, struct iommufd_hw_pagetable, obj);	584	container_of(pt_obj, struct iommufd_hw_pagetable, obj);
376		585
377	rc = iommufd_device_do_attach(idev, hwpt);	586	destroy_hwpt = (*do_attach)(idev, hwpt);
378	if (rc)	587	if (IS_ERR(destroy_hwpt))
379	goto out_put_pt_obj;	588	goto out_put_pt_obj;
380	break;	589	break;
381	}	590	}
@@ -383,26 +592,79 @@ int iommufd_device_attach(struct iommufd_device idev, u32 pt_id) · 1 unmodified line
383	struct iommufd_ioas *ioas =	592	struct iommufd_ioas *ioas =
384	container_of(pt_obj, struct iommufd_ioas, obj);	593	container_of(pt_obj, struct iommufd_ioas, obj);
385		594
386	rc = iommufd_device_auto_get_domain(idev, ioas);	595	destroy_hwpt = iommufd_device_auto_get_domain(idev, ioas, pt_id,
387	if (rc)	596	do_attach);
		597	if (IS_ERR(destroy_hwpt))
388	goto out_put_pt_obj;	598	goto out_put_pt_obj;
389	break;	599	break;
390	}	600	}
391	default:	601	default:
392	rc = -EINVAL;	602	destroy_hwpt = ERR_PTR(-EINVAL);
393	goto out_put_pt_obj;	603	goto out_put_pt_obj;
394	}	604	}
		605	iommufd_put_object(pt_obj);
395		606
396	refcount_inc(&idev->obj.users);	607	/* This destruction has to be after we unlock everything */
397	*pt_id = idev->hwpt->obj.id;	608	if (destroy_hwpt)
398	rc = 0;	609	iommufd_hw_pagetable_put(idev->ictx, destroy_hwpt);
		610	return 0;
399		611
400	out_put_pt_obj:	612	out_put_pt_obj:
401	iommufd_put_object(pt_obj);	613	iommufd_put_object(pt_obj);
402	return rc;	614	return PTR_ERR(destroy_hwpt);
		615	}
		616
		617	/**
		618	* iommufd_device_attach - Connect a device to an iommu_domain
		619	* @idev: device to attach
		620	* @pt_id: Input a IOMMUFD_OBJ_IOAS, or IOMMUFD_OBJ_HW_PAGETABLE
		621	* Output the IOMMUFD_OBJ_HW_PAGETABLE ID
		622	*
		623	* This connects the device to an iommu_domain, either automatically or manually
		624	* selected. Once this completes the device could do DMA.
		625	*
		626	* The caller should return the resulting pt_id back to userspace.
		627	* This function is undone by calling iommufd_device_detach().
		628	*/
		629	int iommufd_device_attach(struct iommufd_device idev, u32 pt_id)
		630	{
		631	int rc;
		632
		633	rc = iommufd_device_change_pt(idev, pt_id, &iommufd_device_do_attach);
		634	if (rc)
		635	return rc;
		636
		637	/*
		638	* Pairs with iommufd_device_detach() - catches caller bugs attempting
		639	* to destroy a device with an attachment.
		640	*/
		641	refcount_inc(&idev->obj.users);
		642	return 0;
403	}	643	}
404	EXPORT_SYMBOL_NS_GPL(iommufd_device_attach, IOMMUFD);	644	EXPORT_SYMBOL_NS_GPL(iommufd_device_attach, IOMMUFD);
405		645
		646	/**
		647	* iommufd_device_replace - Change the device's iommu_domain
		648	* @idev: device to change
		649	* @pt_id: Input a IOMMUFD_OBJ_IOAS, or IOMMUFD_OBJ_HW_PAGETABLE
		650	* Output the IOMMUFD_OBJ_HW_PAGETABLE ID
		651	*
		652	* This is the same as::
		653	*
		654	* iommufd_device_detach();
		655	* iommufd_device_attach();
		656	*
		657	* If it fails then no change is made to the attachment. The iommu driver may
		658	* implement this so there is no disruption in translation. This can only be
		659	* called if iommufd_device_attach() has already succeeded.
		660	*/
		661	int iommufd_device_replace(struct iommufd_device idev, u32 pt_id)
		662	{
		663	return iommufd_device_change_pt(idev, pt_id,
		664	&iommufd_device_do_replace);
		665	}
		666	EXPORT_SYMBOL_NS_GPL(iommufd_device_replace, IOMMUFD);
		667
406	/**	668	/**
407	* iommufd_device_detach - Disconnect a device to an iommu_domain	669	* iommufd_device_detach - Disconnect a device to an iommu_domain
408	* @idev: device to detach	670	* @idev: device to detach
@@ -412,33 +674,87 @@ EXPORT_SYMBOL_NS_GPL(iommufd_device_attach, IOMMUFD); · 3 unmodified lines
412	*/	674	*/
413	void iommufd_device_detach(struct iommufd_device *idev)	675	void iommufd_device_detach(struct iommufd_device *idev)
414	{	676	{
415	struct iommufd_hw_pagetable *hwpt = idev->hwpt;	677	struct iommufd_hw_pagetable *hwpt;
416
417	mutex_lock(&hwpt->devices_lock);
418	list_del(&idev->devices_item);
419	idev->hwpt = NULL;
420	iommufd_hw_pagetable_detach(hwpt, idev);
421	mutex_unlock(&hwpt->devices_lock);
422
423	if (hwpt->auto_domain)
424	iommufd_object_deref_user(idev->ictx, &hwpt->obj);
425	else
426	refcount_dec(&hwpt->obj.users);
427		678
		679	hwpt = iommufd_hw_pagetable_detach(idev);
		680	iommufd_hw_pagetable_put(idev->ictx, hwpt);
428	refcount_dec(&idev->obj.users);	681	refcount_dec(&idev->obj.users);
429	}	682	}
430	EXPORT_SYMBOL_NS_GPL(iommufd_device_detach, IOMMUFD);	683	EXPORT_SYMBOL_NS_GPL(iommufd_device_detach, IOMMUFD);
431		684
		685	/*
		686	* On success, it will refcount_inc() at a valid new_ioas and refcount_dec() at
		687	* a valid cur_ioas (access->ioas). A caller passing in a valid new_ioas should
		688	* call iommufd_put_object() if it does an iommufd_get_object() for a new_ioas.
		689	*/
		690	static int iommufd_access_change_ioas(struct iommufd_access *access,
		691	struct iommufd_ioas *new_ioas)
		692	{
		693	u32 iopt_access_list_id = access->iopt_access_list_id;
		694	struct iommufd_ioas *cur_ioas = access->ioas;
		695	int rc;
		696
		697	lockdep_assert_held(&access->ioas_lock);
		698
		699	/* We are racing with a concurrent detach, bail */
		700	if (cur_ioas != access->ioas_unpin)
		701	return -EBUSY;
		702
		703	if (cur_ioas == new_ioas)
		704	return 0;
		705
		706	/*
		707	* Set ioas to NULL to block any further iommufd_access_pin_pages().
		708	* iommufd_access_unpin_pages() can continue using access->ioas_unpin.
		709	*/
		710	access->ioas = NULL;
		711
		712	if (new_ioas) {
		713	rc = iopt_add_access(&new_ioas->iopt, access);
		714	if (rc) {
		715	access->ioas = cur_ioas;
		716	return rc;
		717	}
		718	refcount_inc(&new_ioas->obj.users);
		719	}
		720
		721	if (cur_ioas) {
		722	if (access->ops->unmap) {
		723	mutex_unlock(&access->ioas_lock);
		724	access->ops->unmap(access->data, 0, ULONG_MAX);
		725	mutex_lock(&access->ioas_lock);
		726	}
		727	iopt_remove_access(&cur_ioas->iopt, access, iopt_access_list_id);
		728	refcount_dec(&cur_ioas->obj.users);
		729	}
		730
		731	access->ioas = new_ioas;
		732	access->ioas_unpin = new_ioas;
		733
		734	return 0;
		735	}
		736
		737	static int iommufd_access_change_ioas_id(struct iommufd_access *access, u32 id)
		738	{
		739	struct iommufd_ioas *ioas = iommufd_get_ioas(access->ictx, id);
		740	int rc;
		741
		742	if (IS_ERR(ioas))
		743	return PTR_ERR(ioas);
		744	rc = iommufd_access_change_ioas(access, ioas);
		745	iommufd_put_object(&ioas->obj);
		746	return rc;
		747	}
		748
432	void iommufd_access_destroy_object(struct iommufd_object *obj)	749	void iommufd_access_destroy_object(struct iommufd_object *obj)
433	{	750	{
434	struct iommufd_access *access =	751	struct iommufd_access *access =
435	container_of(obj, struct iommufd_access, obj);	752	container_of(obj, struct iommufd_access, obj);
436		753
437	if (access->ioas) {	754	mutex_lock(&access->ioas_lock);
438	iopt_remove_access(&access->ioas->iopt, access);	755	if (access->ioas)
439	refcount_dec(&access->ioas->obj.users);	756	WARN_ON(iommufd_access_change_ioas(access, NULL));
440	access->ioas = NULL;	757	mutex_unlock(&access->ioas_lock);
441	}
442	iommufd_ctx_put(access->ictx);	758	iommufd_ctx_put(access->ictx);
443	}	759	}
444		760
@@ -502,61 +818,46 @@ EXPORT_SYMBOL_NS_GPL(iommufd_access_destroy, IOMMUFD); · 57 unmodified lines
502		818
503	void iommufd_access_detach(struct iommufd_access *access)	819	void iommufd_access_detach(struct iommufd_access *access)
504	{	820	{
505	struct iommufd_ioas *cur_ioas = access->ioas;
506
507	mutex_lock(&access->ioas_lock);	821	mutex_lock(&access->ioas_lock);
508	if (WARN_ON(!access->ioas))	822	if (WARN_ON(!access->ioas)) {
509	goto out;
510	/*
511	* Set ioas to NULL to block any further iommufd_access_pin_pages().
512	* iommufd_access_unpin_pages() can continue using access->ioas_unpin.
513	*/
514	access->ioas = NULL;
515
516	if (access->ops->unmap) {
517	mutex_unlock(&access->ioas_lock);	823	mutex_unlock(&access->ioas_lock);
518	access->ops->unmap(access->data, 0, ULONG_MAX);	824	return;
519	mutex_lock(&access->ioas_lock);
520	}	825	}
521	iopt_remove_access(&cur_ioas->iopt, access);	826	WARN_ON(iommufd_access_change_ioas(access, NULL));
522	refcount_dec(&cur_ioas->obj.users);
523	out:
524	access->ioas_unpin = NULL;
525	mutex_unlock(&access->ioas_lock);	827	mutex_unlock(&access->ioas_lock);
526	}	828	}
527	EXPORT_SYMBOL_NS_GPL(iommufd_access_detach, IOMMUFD);	829	EXPORT_SYMBOL_NS_GPL(iommufd_access_detach, IOMMUFD);
528		830
529	int iommufd_access_attach(struct iommufd_access *access, u32 ioas_id)	831	int iommufd_access_attach(struct iommufd_access *access, u32 ioas_id)
530	{	832	{
531	struct iommufd_ioas *new_ioas;	833	int rc;
532	int rc = 0;
533		834
534	mutex_lock(&access->ioas_lock);	835	mutex_lock(&access->ioas_lock);
535	if (WARN_ON(access->ioas \|\| access->ioas_unpin)) {	836	if (WARN_ON(access->ioas)) {
536	mutex_unlock(&access->ioas_lock);	837	mutex_unlock(&access->ioas_lock);
537	return -EINVAL;	838	return -EINVAL;
538	}	839	}
539		840
540	new_ioas = iommufd_get_ioas(access->ictx, ioas_id);	841	rc = iommufd_access_change_ioas_id(access, ioas_id);
541	if (IS_ERR(new_ioas)) {	842	mutex_unlock(&access->ioas_lock);
542	mutex_unlock(&access->ioas_lock);	843	return rc;
543	return PTR_ERR(new_ioas);	844	}
544	}	845	EXPORT_SYMBOL_NS_GPL(iommufd_access_attach, IOMMUFD);
		846
		847	int iommufd_access_replace(struct iommufd_access *access, u32 ioas_id)
		848	{
		849	int rc;
545		850
546	rc = iopt_add_access(&new_ioas->iopt, access);	851	mutex_lock(&access->ioas_lock);
547	if (rc) {	852	if (!access->ioas) {
548	mutex_unlock(&access->ioas_lock);	853	mutex_unlock(&access->ioas_lock);
549	iommufd_put_object(&new_ioas->obj);	854	return -ENOENT;
550	return rc;
551	}	855	}
552	iommufd_ref_to_users(&new_ioas->obj);	856	rc = iommufd_access_change_ioas_id(access, ioas_id);
553
554	access->ioas = new_ioas;
555	access->ioas_unpin = new_ioas;
556	mutex_unlock(&access->ioas_lock);	857	mutex_unlock(&access->ioas_lock);
557	return 0;	858	return rc;
558	}	859	}
559	EXPORT_SYMBOL_NS_GPL(iommufd_access_attach, IOMMUFD);	860	EXPORT_SYMBOL_NS_GPL(iommufd_access_replace, IOMMUFD);
560		861
561	/**	862	/**
562	* iommufd_access_notify_unmap - Notify users of an iopt to stop using it	863	* iommufd_access_notify_unmap - Notify users of an iopt to stop using it
@@ -819,3 +1120,75 @@ err_out: · 256 unmodified lines
819	return rc;	1120	return rc;
820	}	1121	}
821	EXPORT_SYMBOL_NS_GPL(iommufd_access_rw, IOMMUFD);	1122	EXPORT_SYMBOL_NS_GPL(iommufd_access_rw, IOMMUFD);
		1123
		1124	int iommufd_get_hw_info(struct iommufd_ucmd *ucmd)
		1125	{
		1126	struct iommu_hw_info *cmd = ucmd->cmd;
		1127	void __user *user_ptr = u64_to_user_ptr(cmd->data_uptr);
		1128	const struct iommu_ops *ops;
		1129	struct iommufd_device *idev;
		1130	unsigned int data_len;
		1131	unsigned int copy_len;
		1132	void *data;
		1133	int rc;
		1134
		1135	if (cmd->flags \|\| cmd->__reserved)
		1136	return -EOPNOTSUPP;
		1137
		1138	idev = iommufd_get_device(ucmd, cmd->dev_id);
		1139	if (IS_ERR(idev))
		1140	return PTR_ERR(idev);
		1141
		1142	ops = dev_iommu_ops(idev->dev);
		1143	if (ops->hw_info) {
		1144	data = ops->hw_info(idev->dev, &data_len, &cmd->out_data_type);
		1145	if (IS_ERR(data)) {
		1146	rc = PTR_ERR(data);
		1147	goto out_put;
		1148	}
		1149
		1150	/*
		1151	* drivers that have hw_info callback should have a unique
		1152	* iommu_hw_info_type.
		1153	*/
		1154	if (WARN_ON_ONCE(cmd->out_data_type ==
		1155	IOMMU_HW_INFO_TYPE_NONE)) {
		1156	rc = -ENODEV;
		1157	goto out_free;
		1158	}
		1159	} else {
		1160	cmd->out_data_type = IOMMU_HW_INFO_TYPE_NONE;
		1161	data_len = 0;
		1162	data = NULL;
		1163	}
		1164
		1165	copy_len = min(cmd->data_len, data_len);
		1166	if (copy_to_user(user_ptr, data, copy_len)) {
		1167	rc = -EFAULT;
		1168	goto out_free;
		1169	}
		1170
		1171	/*
		1172	* Zero the trailing bytes if the user buffer is bigger than the
		1173	* data size kernel actually has.
		1174	*/
		1175	if (copy_len < cmd->data_len) {
		1176	if (clear_user(user_ptr + copy_len, cmd->data_len - copy_len)) {
		1177	rc = -EFAULT;
		1178	goto out_free;
		1179	}
		1180	}
		1181
		1182	/*
		1183	* We return the length the kernel supports so userspace may know what
		1184	* the kernel capability is. It could be larger than the input buffer.
		1185	*/
		1186	cmd->data_len = data_len;
		1187
		1188	rc = iommufd_ucmd_respond(ucmd, sizeof(*cmd));
		1189	out_free:
		1190	kfree(data);
		1191	out_put:
		1192	iommufd_put_object(&idev->obj);
		1193	return rc;
		1194	}

drivers/iommu/iommufd/hw_pagetable.c+93−19

@@ -3,6 +3,7 @@ · 2 unmodified lines
3	* Copyright (c) 2021-2022, NVIDIA CORPORATION & AFFILIATES	3	* Copyright (c) 2021-2022, NVIDIA CORPORATION & AFFILIATES
4	*/	4	*/
5	#include <linux/iommu.h>	5	#include <linux/iommu.h>
		6	#include <uapi/linux/iommufd.h>
6		7
7	#include "iommufd_private.h"	8	#include "iommufd_private.h"
8		9
@@ -11,8 +12,6 @@ void iommufd_hw_pagetable_destroy(struct iommufd_object *obj) · 2 unmodified lines
11	struct iommufd_hw_pagetable *hwpt =	12	struct iommufd_hw_pagetable *hwpt =
12	container_of(obj, struct iommufd_hw_pagetable, obj);	13	container_of(obj, struct iommufd_hw_pagetable, obj);
13		14
14	WARN_ON(!list_empty(&hwpt->devices));
15
16	if (!list_empty(&hwpt->hwpt_item)) {	15	if (!list_empty(&hwpt->hwpt_item)) {
17	mutex_lock(&hwpt->ioas->mutex);	16	mutex_lock(&hwpt->ioas->mutex);
18	list_del(&hwpt->hwpt_item);	17	list_del(&hwpt->hwpt_item);
@@ -25,7 +24,35 @@ void iommufd_hw_pagetable_destroy(struct iommufd_object *obj) · 6 unmodified lines
25	iommu_domain_free(hwpt->domain);	24	iommu_domain_free(hwpt->domain);
26		25
27	refcount_dec(&hwpt->ioas->obj.users);	26	refcount_dec(&hwpt->ioas->obj.users);
28	mutex_destroy(&hwpt->devices_lock);	27	}
		28
		29	void iommufd_hw_pagetable_abort(struct iommufd_object *obj)
		30	{
		31	struct iommufd_hw_pagetable *hwpt =
		32	container_of(obj, struct iommufd_hw_pagetable, obj);
		33
		34	/* The ioas->mutex must be held until finalize is called. */
		35	lockdep_assert_held(&hwpt->ioas->mutex);
		36
		37	if (!list_empty(&hwpt->hwpt_item)) {
		38	list_del_init(&hwpt->hwpt_item);
		39	iopt_table_remove_domain(&hwpt->ioas->iopt, hwpt->domain);
		40	}
		41	iommufd_hw_pagetable_destroy(obj);
		42	}
		43
		44	int iommufd_hw_pagetable_enforce_cc(struct iommufd_hw_pagetable *hwpt)
		45	{
		46	if (hwpt->enforce_cache_coherency)
		47	return 0;
		48
		49	if (hwpt->domain->ops->enforce_cache_coherency)
		50	hwpt->enforce_cache_coherency =
		51	hwpt->domain->ops->enforce_cache_coherency(
		52	hwpt->domain);
		53	if (!hwpt->enforce_cache_coherency)
		54	return -EINVAL;
		55	return 0;
29	}	56	}
30		57
31	/**	58	/**
@@ -38,6 +65,10 @@ void iommufd_hw_pagetable_destroy(struct iommufd_object *obj) · 6 unmodified lines
38	* Allocate a new iommu_domain and return it as a hw_pagetable. The HWPT	65	* Allocate a new iommu_domain and return it as a hw_pagetable. The HWPT
39	* will be linked to the given ioas and upon return the underlying iommu_domain	66	* will be linked to the given ioas and upon return the underlying iommu_domain
40	* is fully popoulated.	67	* is fully popoulated.
		68	*
		69	* The caller must hold the ioas->mutex until after
		70	* iommufd_object_abort_and_destroy() or iommufd_object_finalize() is called on
		71	* the returned hwpt.
41	*/	72	*/
42	struct iommufd_hw_pagetable *	73	struct iommufd_hw_pagetable *
43	iommufd_hw_pagetable_alloc(struct iommufd_ctx ictx, struct iommufd_ioas ioas,	74	iommufd_hw_pagetable_alloc(struct iommufd_ctx ictx, struct iommufd_ioas ioas,
@@ -52,9 +83,7 @@ iommufd_hw_pagetable_alloc(struct iommufd_ctx ictx, struct iommufd_ioas ioas, · 8 unmodified lines
52	if (IS_ERR(hwpt))	83	if (IS_ERR(hwpt))
53	return hwpt;	84	return hwpt;
54		85
55	INIT_LIST_HEAD(&hwpt->devices);
56	INIT_LIST_HEAD(&hwpt->hwpt_item);	86	INIT_LIST_HEAD(&hwpt->hwpt_item);
57	mutex_init(&hwpt->devices_lock);
58	/* Pairs with iommufd_hw_pagetable_destroy() */	87	/* Pairs with iommufd_hw_pagetable_destroy() */
59	refcount_inc(&ioas->obj.users);	88	refcount_inc(&ioas->obj.users);
60	hwpt->ioas = ioas;	89	hwpt->ioas = ioas;
@@ -65,7 +94,18 @@ iommufd_hw_pagetable_alloc(struct iommufd_ctx ictx, struct iommufd_ioas ioas, · 4 unmodified lines
65	goto out_abort;	94	goto out_abort;
66	}	95	}
67		96
68	mutex_lock(&hwpt->devices_lock);	97	/*
		98	* Set the coherency mode before we do iopt_table_add_domain() as some
		99	* iommus have a per-PTE bit that controls it and need to decide before
		100	* doing any maps. It is an iommu driver bug to report
		101	* IOMMU_CAP_ENFORCE_CACHE_COHERENCY but fail enforce_cache_coherency on
		102	* a new domain.
		103	*/
		104	if (idev->enforce_cache_coherency) {
		105	rc = iommufd_hw_pagetable_enforce_cc(hwpt);
		106	if (WARN_ON(rc))
		107	goto out_abort;
		108	}
69		109
70	/*	110	/*
71	* immediate_attach exists only to accommodate iommu drivers that cannot	111	* immediate_attach exists only to accommodate iommu drivers that cannot
@@ -76,30 +116,64 @@ iommufd_hw_pagetable_alloc(struct iommufd_ctx ictx, struct iommufd_ioas ioas, · 4 unmodified lines
76	if (immediate_attach) {	116	if (immediate_attach) {
77	rc = iommufd_hw_pagetable_attach(hwpt, idev);	117	rc = iommufd_hw_pagetable_attach(hwpt, idev);
78	if (rc)	118	if (rc)
79	goto out_unlock;	119	goto out_abort;
80	}	120	}
81		121
82	rc = iopt_table_add_domain(&hwpt->ioas->iopt, hwpt->domain);	122	rc = iopt_table_add_domain(&hwpt->ioas->iopt, hwpt->domain);
83	if (rc)	123	if (rc)
84	goto out_detach;	124	goto out_detach;
85	list_add_tail(&hwpt->hwpt_item, &hwpt->ioas->hwpt_list);	125	list_add_tail(&hwpt->hwpt_item, &hwpt->ioas->hwpt_list);
86
87	if (immediate_attach) {
88	/* See iommufd_device_do_attach() */
89	refcount_inc(&hwpt->obj.users);
90	idev->hwpt = hwpt;
91	list_add(&idev->devices_item, &hwpt->devices);
92	}
93
94	mutex_unlock(&hwpt->devices_lock);
95	return hwpt;	126	return hwpt;
96		127
97	out_detach:	128	out_detach:
98	if (immediate_attach)	129	if (immediate_attach)
99	iommufd_hw_pagetable_detach(hwpt, idev);	130	iommufd_hw_pagetable_detach(idev);
100	out_unlock:
101	mutex_unlock(&hwpt->devices_lock);
102	out_abort:	131	out_abort:
103	iommufd_object_abort_and_destroy(ictx, &hwpt->obj);	132	iommufd_object_abort_and_destroy(ictx, &hwpt->obj);
104	return ERR_PTR(rc);	133	return ERR_PTR(rc);
105	}	134	}
		135
		136	int iommufd_hwpt_alloc(struct iommufd_ucmd *ucmd)
		137	{
		138	struct iommu_hwpt_alloc *cmd = ucmd->cmd;
		139	struct iommufd_hw_pagetable *hwpt;
		140	struct iommufd_device *idev;
		141	struct iommufd_ioas *ioas;
		142	int rc;
		143
		144	if (cmd->flags \|\| cmd->__reserved)
		145	return -EOPNOTSUPP;
		146
		147	idev = iommufd_get_device(ucmd, cmd->dev_id);
		148	if (IS_ERR(idev))
		149	return PTR_ERR(idev);
		150
		151	ioas = iommufd_get_ioas(ucmd->ictx, cmd->pt_id);
		152	if (IS_ERR(ioas)) {
		153	rc = PTR_ERR(ioas);
		154	goto out_put_idev;
		155	}
		156
		157	mutex_lock(&ioas->mutex);
		158	hwpt = iommufd_hw_pagetable_alloc(ucmd->ictx, ioas, idev, false);
		159	if (IS_ERR(hwpt)) {
		160	rc = PTR_ERR(hwpt);
		161	goto out_unlock;
		162	}
		163
		164	cmd->out_hwpt_id = hwpt->obj.id;
		165	rc = iommufd_ucmd_respond(ucmd, sizeof(*cmd));
		166	if (rc)
		167	goto out_hwpt;
		168	iommufd_object_finalize(ucmd->ictx, &hwpt->obj);
		169	goto out_unlock;
		170
		171	out_hwpt:
		172	iommufd_object_abort_and_destroy(ucmd->ictx, &hwpt->obj);
		173	out_unlock:
		174	mutex_unlock(&ioas->mutex);
		175	iommufd_put_object(&ioas->obj);
		176	out_put_idev:
		177	iommufd_put_object(&idev->obj);
		178	return rc;
		179	}

drivers/iommu/iommufd/io_pagetable.c+17−19

 }
 void iopt_remove_access(struct io_pagetable *iopt,
-			struct iommufd_access *access)
+			struct iommufd_access *access,
+			u32 iopt_access_list_id)
 {
 	down_write(&iopt->domains_rwsem);
 	down_write(&iopt->iova_rwsem);
-	WARN_ON(xa_erase(&iopt->access_list, access->iopt_access_list_id) !=
+	WARN_ON(xa_erase(&iopt->access_list, iopt_access_list_id) != access);
-		access);
 	WARN_ON(iopt_calculate_iova_alignment(iopt));
 	up_write(&iopt->iova_rwsem);
 	up_write(&iopt->domains_rwsem);
 }
-/* Narrow the valid_iova_itree to include reserved ranges from a group. */
+/* Narrow the valid_iova_itree to include reserved ranges from a device. */
-int iopt_table_enforce_group_resv_regions(struct io_pagetable *iopt,
+int iopt_table_enforce_dev_resv_regions(struct io_pagetable *iopt,
-					  struct device *device,
+					struct device *dev,
-					  struct iommu_group *group,
+					phys_addr_t *sw_msi_start)
-					  phys_addr_t *sw_msi_start)
 {
 	struct iommu_resv_region *resv;
-	struct iommu_resv_region *tmp;
+	LIST_HEAD(resv_regions);
-	LIST_HEAD(group_resv_regions);
 	unsigned int num_hw_msi = 0;
 	unsigned int num_sw_msi = 0;
 	int rc;
+	if (iommufd_should_fail())
+		return -EINVAL;
 	down_write(&iopt->iova_rwsem);
-	rc = iommu_get_group_resv_regions(group, &group_resv_regions);
+	/* FIXME: drivers allocate memory but there is no failure propogated */
-	if (rc)
+	iommu_get_resv_regions(dev, &resv_regions);
-		goto out_unlock;
-	list_for_each_entry(resv, &group_resv_regions, list) {
+	list_for_each_entry(resv, &resv_regions, list) {
 		if (resv->type == IOMMU_RESV_DIRECT_RELAXABLE)
 			continue;
 		}
 		rc = iopt_reserve_iova(iopt, resv->start,
-				       resv->length - 1 + resv->start, device);
+				       resv->length - 1 + resv->start, dev);
 		if (rc)
 			goto out_reserved;
 	}
 	goto out_free_resv;
 out_reserved:
-	__iopt_remove_reserved_iova(iopt, device);
+	__iopt_remove_reserved_iova(iopt, dev);
 out_free_resv:
-	list_for_each_entry_safe(resv, tmp, &group_resv_regions, list)
+	iommu_put_resv_regions(dev, &resv_regions);
-		kfree(resv);
-out_unlock:
 	up_write(&iopt->iova_rwsem);
 	return rc;
 }

drivers/iommu/iommufd/iommufd_private.h+46−38

 struct iommufd_ctx {
 	struct file *file;
 	struct xarray objects;
+	struct xarray groups;
 	u8 account_mode;
 	/* Compatibility with VFIO no iommu */
 			  struct iommu_domain *domain);
 void iopt_table_remove_domain(struct io_pagetable *iopt,
 			      struct iommu_domain *domain);
-int iopt_table_enforce_group_resv_regions(struct io_pagetable *iopt,
+int iopt_table_enforce_dev_resv_regions(struct io_pagetable *iopt,
-					  struct device *device,
+					struct device *dev,
-					  struct iommu_group *group,
+					phys_addr_t *sw_msi_start);
-					  phys_addr_t *sw_msi_start);
 int iopt_set_allow_iova(struct io_pagetable *iopt,
 			struct rb_root_cached *allowed_iova);
 int iopt_reserve_iova(struct io_pagetable *iopt, unsigned long start,
 #ifdef CONFIG_IOMMUFD_TEST
 	IOMMUFD_OBJ_SELFTEST,
 #endif
+	IOMMUFD_OBJ_MAX,
 };
 /* Base struct for all objects with a userspace ID handle. */
 	up_read(&obj->destroy_rwsem);
 }
-/**
- * iommufd_ref_to_users() - Switch from destroy_rwsem to users refcount
- *        protection
- * @obj - Object to release
- *
- * Objects have two refcount protections (destroy_rwsem and the refcount_t
- * users). Holding either of these will prevent the object from being destroyed.
- *
- * Depending on the use case, one protection or the other is appropriate.  In
- * most cases references are being protected by the destroy_rwsem. This allows
- * orderly destruction of the object because iommufd_object_destroy_user() will
- * wait for it to become unlocked. However, as a rwsem, it cannot be held across
- * a system call return. So cases that have longer term needs must switch
- * to the weaker users refcount_t.
- *
- * With users protection iommufd_object_destroy_user() will return false,
- * refusing to destroy the object, causing -EBUSY to userspace.
- */
-static inline void iommufd_ref_to_users(struct iommufd_object *obj)
-{
-	up_read(&obj->destroy_rwsem);
-	/* iommufd_lock_obj() obtains users as well */
-}
 void iommufd_object_abort(struct iommufd_ctx *ictx, struct iommufd_object *obj);
 void iommufd_object_abort_and_destroy(struct iommufd_ctx *ictx,
 				      struct iommufd_object *obj);
 	bool msi_cookie : 1;
 	/* Head at iommufd_ioas::hwpt_list */
 	struct list_head hwpt_item;
-	struct mutex devices_lock;
-	struct list_head devices;
 };
 struct iommufd_hw_pagetable *
 iommufd_hw_pagetable_alloc(struct iommufd_ctx *ictx, struct iommufd_ioas *ioas,
 			   struct iommufd_device *idev, bool immediate_attach);
+int iommufd_hw_pagetable_enforce_cc(struct iommufd_hw_pagetable *hwpt);
 int iommufd_hw_pagetable_attach(struct iommufd_hw_pagetable *hwpt,
 				struct iommufd_device *idev);
-void iommufd_hw_pagetable_detach(struct iommufd_hw_pagetable *hwpt,
+struct iommufd_hw_pagetable *
-				 struct iommufd_device *idev);
+iommufd_hw_pagetable_detach(struct iommufd_device *idev);
 void iommufd_hw_pagetable_destroy(struct iommufd_object *obj);
+void iommufd_hw_pagetable_abort(struct iommufd_object *obj);
+int iommufd_hwpt_alloc(struct iommufd_ucmd *ucmd);
+static inline void iommufd_hw_pagetable_put(struct iommufd_ctx *ictx,
+					    struct iommufd_hw_pagetable *hwpt)
+{
+	lockdep_assert_not_held(&hwpt->ioas->mutex);
+	if (hwpt->auto_domain)
+		iommufd_object_deref_user(ictx, &hwpt->obj);
+	else
+		refcount_dec(&hwpt->obj.users);
+}
+struct iommufd_group {
+	struct kref ref;
+	struct mutex lock;
+	struct iommufd_ctx *ictx;
+	struct iommu_group *group;
+	struct iommufd_hw_pagetable *hwpt;
+	struct list_head device_list;
+	phys_addr_t sw_msi_start;
+};
 /*
  * A iommufd_device object represents the binding relationship between a
 struct iommufd_device {
 	struct iommufd_object obj;
 	struct iommufd_ctx *ictx;
-	struct iommufd_hw_pagetable *hwpt;
+	struct iommufd_group *igroup;
-	/* Head at iommufd_hw_pagetable::devices */
+	struct list_head group_item;
-	struct list_head devices_item;
 	/* always the physical device */
 	struct device *dev;
-	struct iommu_group *group;
 	bool enforce_cache_coherency;
 };
+static inline struct iommufd_device *
+iommufd_get_device(struct iommufd_ucmd *ucmd, u32 id)
+{
+	return container_of(iommufd_get_object(ucmd->ictx, id,
+					       IOMMUFD_OBJ_DEVICE),
+			    struct iommufd_device, obj);
+}
 void iommufd_device_destroy(struct iommufd_object *obj);
+int iommufd_get_hw_info(struct iommufd_ucmd *ucmd);
 struct iommufd_access {
 	struct iommufd_object obj;
 int iopt_add_access(struct io_pagetable *iopt, struct iommufd_access *access);
 void iopt_remove_access(struct io_pagetable *iopt,
-			struct iommufd_access *access);
+			struct iommufd_access *access,
+			u32 iopt_access_list_id);
 void iommufd_access_destroy_object(struct iommufd_object *obj);
 #ifdef CONFIG_IOMMUFD_TEST
 void iommufd_test_syz_conv_iova_id(struct iommufd_ucmd *ucmd,
 				   unsigned int ioas_id, u64 *iova, u32 *flags);
 bool iommufd_should_fail(void);
-void __init iommufd_test_init(void);
+int __init iommufd_test_init(void);
 void iommufd_test_exit(void);
 bool iommufd_selftest_is_mock_dev(struct device *dev);
 #else
 {
 	return false;
 }
-static inline void __init iommufd_test_init(void)
+static inline int __init iommufd_test_init(void)
 {
+	return 0;
 }
 static inline void iommufd_test_exit(void)
 {

drivers/iommu/iommufd/iommufd_test.h+19−0

 	IOMMU_TEST_OP_ACCESS_PAGES,
 	IOMMU_TEST_OP_ACCESS_RW,
 	IOMMU_TEST_OP_SET_TEMP_MEMORY_LIMIT,
+	IOMMU_TEST_OP_MOCK_DOMAIN_REPLACE,
+	IOMMU_TEST_OP_ACCESS_REPLACE_IOAS,
 };
 enum {
 		struct {
 			__u32 out_stdev_id;
 			__u32 out_hwpt_id;
+			/* out_idev_id is the standard iommufd_bind object */
+			__u32 out_idev_id;
 		} mock_domain;
+		struct {
+			__u32 pt_id;
+		} mock_domain_replace;
 		struct {
 			__aligned_u64 iova;
 			__aligned_u64 length;
 		struct {
 			__u32 limit;
 		} memory_limit;
+		struct {
+			__u32 ioas_id;
+		} access_replace_ioas;
 	};
 	__u32 last;
 };
 #define IOMMU_TEST_CMD _IO(IOMMUFD_TYPE, IOMMUFD_CMD_BASE + 32)
+/* Mock structs for IOMMU_DEVICE_GET_HW_INFO ioctl */
+#define IOMMU_HW_INFO_TYPE_SELFTEST	0xfeedbeef
+#define IOMMU_HW_INFO_SELFTEST_REGVAL	0xdeadbeef
+struct iommu_test_hw_info {
+	__u32 flags;
+	__u32 test_reg;
+};
 #endif

drivers/iommu/iommufd/main.c+32−3

 struct iommufd_object_ops {
 	void (*destroy)(struct iommufd_object *obj);
+	void (*abort)(struct iommufd_object *obj);
 };
 static const struct iommufd_object_ops iommufd_object_ops[];
 static struct miscdevice vfio_misc_dev;
 					     size_t size,
 					     enum iommufd_object_type type)
 {
+	static struct lock_class_key obj_keys[IOMMUFD_OBJ_MAX];
 	struct iommufd_object *obj;
 	int rc;
 	if (!obj)
 		return ERR_PTR(-ENOMEM);
 	obj->type = type;
-	init_rwsem(&obj->destroy_rwsem);
+	/*
+	 * In most cases the destroy_rwsem is obtained with try so it doesn't
+	 * interact with lockdep, however on destroy we have to sleep. This
+	 * means if we have to destroy an object while holding a get on another
+	 * object it triggers lockdep. Using one locking class per object type
+	 * is a simple and reasonable way to avoid this.
+	 */
+	__init_rwsem(&obj->destroy_rwsem, "iommufd_object::destroy_rwsem",
+		     &obj_keys[type]);
 	refcount_set(&obj->users, 1);
 	/*
 void iommufd_object_abort_and_destroy(struct iommufd_ctx *ictx,
 				      struct iommufd_object *obj)
 {
-	iommufd_object_ops[obj->type].destroy(obj);
+	if (iommufd_object_ops[obj->type].abort)
+		iommufd_object_ops[obj->type].abort(obj);
+	else
+		iommufd_object_ops[obj->type].destroy(obj);
 	iommufd_object_abort(ictx, obj);
 }
 	}
 	xa_init_flags(&ictx->objects, XA_FLAGS_ALLOC1 | XA_FLAGS_ACCOUNT);
+	xa_init(&ictx->groups);
 	ictx->file = filp;
 	filp->private_data = ictx;
 	return 0;
 		if (WARN_ON(!destroyed))
 			break;
 	}
+	WARN_ON(!xa_empty(&ictx->groups));
 	kfree(ictx);
 	return 0;
 }
 union ucmd_buffer {
 	struct iommu_destroy destroy;
+	struct iommu_hw_info info;
+	struct iommu_hwpt_alloc hwpt;
 	struct iommu_ioas_alloc alloc;
 	struct iommu_ioas_allow_iovas allow_iovas;
 	struct iommu_ioas_copy ioas_copy;
 	}
 static const struct iommufd_ioctl_op iommufd_ioctl_ops[] = {
 	IOCTL_OP(IOMMU_DESTROY, iommufd_destroy, struct iommu_destroy, id),
+	IOCTL_OP(IOMMU_GET_HW_INFO, iommufd_get_hw_info, struct iommu_hw_info,
+		 __reserved),
+	IOCTL_OP(IOMMU_HWPT_ALLOC, iommufd_hwpt_alloc, struct iommu_hwpt_alloc,
+		 __reserved),
 	IOCTL_OP(IOMMU_IOAS_ALLOC, iommufd_ioas_alloc_ioctl,
 		 struct iommu_ioas_alloc, out_ioas_id),
 	IOCTL_OP(IOMMU_IOAS_ALLOW_IOVAS, iommufd_ioas_allow_iovas,
 	},
 	[IOMMUFD_OBJ_HW_PAGETABLE] = {
 		.destroy = iommufd_hw_pagetable_destroy,
+		.abort = iommufd_hw_pagetable_abort,
 	},
 #ifdef CONFIG_IOMMUFD_TEST
 	[IOMMUFD_OBJ_SELFTEST] = {
 		if (ret)
 			goto err_misc;
 	}
-	iommufd_test_init();
+	ret = iommufd_test_init();
+	if (ret)
+		goto err_vfio_misc;
 	return 0;
+err_vfio_misc:
+	if (IS_ENABLED(CONFIG_IOMMUFD_VFIO_CONTAINER))
+		misc_deregister(&vfio_misc_dev);
 err_misc:
 	misc_deregister(&iommu_misc_dev);
 	return ret;
 MODULE_ALIAS_MISCDEV(VFIO_MINOR);
 MODULE_ALIAS("devname:vfio/vfio");
 #endif
+MODULE_IMPORT_NS(IOMMUFD_INTERNAL);
 MODULE_DESCRIPTION("I/O Address Space Management for passthrough devices");
 MODULE_LICENSE("GPL");

drivers/iommu/iommufd/selftest.c+157−56

@@ -9,14 +9,17 @@ · 8 unmodified lines
9	#include <linux/file.h>	9	#include <linux/file.h>
10	#include <linux/anon_inodes.h>	10	#include <linux/anon_inodes.h>
11	#include <linux/fault-inject.h>	11	#include <linux/fault-inject.h>
		12	#include <linux/platform_device.h>
12	#include <uapi/linux/iommufd.h>	13	#include <uapi/linux/iommufd.h>
13		14
		15	#include "../iommu-priv.h"
14	#include "io_pagetable.h"	16	#include "io_pagetable.h"
15	#include "iommufd_private.h"	17	#include "iommufd_private.h"
16	#include "iommufd_test.h"	18	#include "iommufd_test.h"
17		19
18	static DECLARE_FAULT_ATTR(fail_iommufd);	20	static DECLARE_FAULT_ATTR(fail_iommufd);
19	static struct dentry *dbgfs_root;	21	static struct dentry *dbgfs_root;
		22	static struct platform_device *selftest_iommu_dev;
20		23
21	size_t iommufd_test_memory_limit = 65536;	24	size_t iommufd_test_memory_limit = 65536;
22		25
@@ -128,6 +131,21 @@ static struct iommu_domain mock_blocking_domain = { · 105 unmodified lines
128	.ops = &mock_blocking_ops,	131	.ops = &mock_blocking_ops,
129	};	132	};
130		133
		134	static void mock_domain_hw_info(struct device dev, u32 length, u32 type)
		135	{
		136	struct iommu_test_hw_info *info;
		137
		138	info = kzalloc(sizeof(*info), GFP_KERNEL);
		139	if (!info)
		140	return ERR_PTR(-ENOMEM);
		141
		142	info->test_reg = IOMMU_HW_INFO_SELFTEST_REGVAL;
		143	length = sizeof(info);
		144	*type = IOMMU_HW_INFO_TYPE_SELFTEST;
		145
		146	return info;
		147	}
		148
131	static struct iommu_domain *mock_domain_alloc(unsigned int iommu_domain_type)	149	static struct iommu_domain *mock_domain_alloc(unsigned int iommu_domain_type)
132	{	150	{
133	struct mock_iommu_domain *mock;	151	struct mock_iommu_domain *mock;
@@ -135,7 +153,7 @@ static struct iommu_domain *mock_domain_alloc(unsigned int iommu_domain_type) · 1 unmodified line
135	if (iommu_domain_type == IOMMU_DOMAIN_BLOCKED)	153	if (iommu_domain_type == IOMMU_DOMAIN_BLOCKED)
136	return &mock_blocking_domain;	154	return &mock_blocking_domain;
137		155
138	if (WARN_ON(iommu_domain_type != IOMMU_DOMAIN_UNMANAGED))	156	if (iommu_domain_type != IOMMU_DOMAIN_UNMANAGED)
139	return NULL;	157	return NULL;
140		158
141	mock = kzalloc(sizeof(*mock), GFP_KERNEL);	159	mock = kzalloc(sizeof(*mock), GFP_KERNEL);
@@ -276,12 +294,23 @@ static void mock_domain_set_plaform_dma_ops(struct device *dev) · 134 unmodified lines
276	*/	294	*/
277	}	295	}
278		296
		297	static struct iommu_device mock_iommu_device = {
		298	};
		299
		300	static struct iommu_device mock_probe_device(struct device dev)
		301	{
		302	return &mock_iommu_device;
		303	}
		304
279	static const struct iommu_ops mock_ops = {	305	static const struct iommu_ops mock_ops = {
280	.owner = THIS_MODULE,	306	.owner = THIS_MODULE,
281	.pgsize_bitmap = MOCK_IO_PAGE_SIZE,	307	.pgsize_bitmap = MOCK_IO_PAGE_SIZE,
		308	.hw_info = mock_domain_hw_info,
282	.domain_alloc = mock_domain_alloc,	309	.domain_alloc = mock_domain_alloc,
283	.capable = mock_domain_capable,	310	.capable = mock_domain_capable,
284	.set_platform_dma_ops = mock_domain_set_plaform_dma_ops,	311	.set_platform_dma_ops = mock_domain_set_plaform_dma_ops,
		312	.device_group = generic_device_group,
		313	.probe_device = mock_probe_device,
285	.default_domain_ops =	314	.default_domain_ops =
286	&(struct iommu_domain_ops){	315	&(struct iommu_domain_ops){
287	.free = mock_domain_free,	316	.free = mock_domain_free,
@@ -292,10 +321,6 @@ static const struct iommu_ops mock_ops = { · 4 unmodified lines
292	},	321	},
293	};	322	};
294		323
295	static struct iommu_device mock_iommu_device = {
296	.ops = &mock_ops,
297	};
298
299	static inline struct iommufd_hw_pagetable *	324	static inline struct iommufd_hw_pagetable *
300	get_md_pagetable(struct iommufd_ucmd *ucmd, u32 mockpt_id,	325	get_md_pagetable(struct iommufd_ucmd *ucmd, u32 mockpt_id,
301	struct mock_iommu_domain **mock)	326	struct mock_iommu_domain **mock)
@@ -316,22 +341,29 @@ get_md_pagetable(struct iommufd_ucmd *ucmd, u32 mockpt_id, · 14 unmodified lines
316	return hwpt;	341	return hwpt;
317	}	342	}
318		343
319	static struct bus_type iommufd_mock_bus_type = {	344	struct mock_bus_type {
320	.name = "iommufd_mock",	345	struct bus_type bus;
321	.iommu_ops = &mock_ops,	346	struct notifier_block nb;
		347	};
		348
		349	static struct mock_bus_type iommufd_mock_bus_type = {
		350	.bus = {
		351	.name = "iommufd_mock",
		352	},
322	};	353	};
323		354
		355	static atomic_t mock_dev_num;
		356
324	static void mock_dev_release(struct device *dev)	357	static void mock_dev_release(struct device *dev)
325	{	358	{
326	struct mock_dev *mdev = container_of(dev, struct mock_dev, dev);	359	struct mock_dev *mdev = container_of(dev, struct mock_dev, dev);
327		360
		361	atomic_dec(&mock_dev_num);
328	kfree(mdev);	362	kfree(mdev);
329	}	363	}
330		364
331	static struct mock_dev *mock_dev_create(void)	365	static struct mock_dev *mock_dev_create(void)
332	{	366	{
333	struct iommu_group *iommu_group;
334	struct dev_iommu *dev_iommu;
335	struct mock_dev *mdev;	367	struct mock_dev *mdev;
336	int rc;	368	int rc;
337		369
@@ -341,51 +373,18 @@ static struct mock_dev *mock_dev_create(void) · 3 unmodified lines
341		373
342	device_initialize(&mdev->dev);	374	device_initialize(&mdev->dev);
343	mdev->dev.release = mock_dev_release;	375	mdev->dev.release = mock_dev_release;
344	mdev->dev.bus = &iommufd_mock_bus_type;	376	mdev->dev.bus = &iommufd_mock_bus_type.bus;
345
346	iommu_group = iommu_group_alloc();
347	if (IS_ERR(iommu_group)) {
348	rc = PTR_ERR(iommu_group);
349	goto err_put;
350	}
351		377
352	rc = dev_set_name(&mdev->dev, "iommufd_mock%u",	378	rc = dev_set_name(&mdev->dev, "iommufd_mock%u",
353	iommu_group_id(iommu_group));	379	atomic_inc_return(&mock_dev_num));
354	if (rc)	380	if (rc)
355	goto err_group;	381	goto err_put;
356
357	/*
358	* The iommu core has no way to associate a single device with an iommu
359	* driver (heck currently it can't even support two iommu_drivers
360	* registering). Hack it together with an open coded dev_iommu_get().
361	* Notice that the normal notifier triggered iommu release process also
362	* does not work here because this bus is not in iommu_buses.
363	*/
364	mdev->dev.iommu = kzalloc(sizeof(*dev_iommu), GFP_KERNEL);
365	if (!mdev->dev.iommu) {
366	rc = -ENOMEM;
367	goto err_group;
368	}
369	mutex_init(&mdev->dev.iommu->lock);
370	mdev->dev.iommu->iommu_dev = &mock_iommu_device;
371		382
372	rc = device_add(&mdev->dev);	383	rc = device_add(&mdev->dev);
373	if (rc)	384	if (rc)
374	goto err_dev_iommu;	385	goto err_put;
375
376	rc = iommu_group_add_device(iommu_group, &mdev->dev);
377	if (rc)
378	goto err_del;
379	iommu_group_put(iommu_group);
380	return mdev;	386	return mdev;
381		387
382	err_del:
383	device_del(&mdev->dev);
384	err_dev_iommu:
385	kfree(mdev->dev.iommu);
386	mdev->dev.iommu = NULL;
387	err_group:
388	iommu_group_put(iommu_group);
389	err_put:	388	err_put:
390	put_device(&mdev->dev);	389	put_device(&mdev->dev);
391	return ERR_PTR(rc);	390	return ERR_PTR(rc);
@@ -393,11 +392,7 @@ err_put: · 1 unmodified line
393		392
394	static void mock_dev_destroy(struct mock_dev *mdev)	393	static void mock_dev_destroy(struct mock_dev *mdev)
395	{	394	{
396	iommu_group_remove_device(&mdev->dev);	395	device_unregister(&mdev->dev);
397	device_del(&mdev->dev);
398	kfree(mdev->dev.iommu);
399	mdev->dev.iommu = NULL;
400	put_device(&mdev->dev);
401	}	396	}
402		397
403	bool iommufd_selftest_is_mock_dev(struct device *dev)	398	bool iommufd_selftest_is_mock_dev(struct device *dev)
@@ -443,9 +438,15 @@ static int iommufd_test_mock_domain(struct iommufd_ucmd *ucmd, · 39 unmodified lines
443	/* Userspace must destroy the device_id to destroy the object */	438	/* Userspace must destroy the device_id to destroy the object */
444	cmd->mock_domain.out_hwpt_id = pt_id;	439	cmd->mock_domain.out_hwpt_id = pt_id;
445	cmd->mock_domain.out_stdev_id = sobj->obj.id;	440	cmd->mock_domain.out_stdev_id = sobj->obj.id;
		441	cmd->mock_domain.out_idev_id = idev_id;
		442	rc = iommufd_ucmd_respond(ucmd, sizeof(*cmd));
		443	if (rc)
		444	goto out_detach;
446	iommufd_object_finalize(ucmd->ictx, &sobj->obj);	445	iommufd_object_finalize(ucmd->ictx, &sobj->obj);
447	return iommufd_ucmd_respond(ucmd, sizeof(*cmd));	446	return 0;
448		447
		448	out_detach:
		449	iommufd_device_detach(idev);
449	out_unbind:	450	out_unbind:
450	iommufd_device_unbind(idev);	451	iommufd_device_unbind(idev);
451	out_mdev:	452	out_mdev:
@@ -455,6 +456,42 @@ out_sobj: · 3 unmodified lines
455	return rc;	456	return rc;
456	}	457	}
457		458
		459	/* Replace the mock domain with a manually allocated hw_pagetable */
		460	static int iommufd_test_mock_domain_replace(struct iommufd_ucmd *ucmd,
		461	unsigned int device_id, u32 pt_id,
		462	struct iommu_test_cmd *cmd)
		463	{
		464	struct iommufd_object *dev_obj;
		465	struct selftest_obj *sobj;
		466	int rc;
		467
		468	/*
		469	* Prefer to use the OBJ_SELFTEST because the destroy_rwsem will ensure
		470	* it doesn't race with detach, which is not allowed.
		471	*/
		472	dev_obj =
		473	iommufd_get_object(ucmd->ictx, device_id, IOMMUFD_OBJ_SELFTEST);
		474	if (IS_ERR(dev_obj))
		475	return PTR_ERR(dev_obj);
		476
		477	sobj = container_of(dev_obj, struct selftest_obj, obj);
		478	if (sobj->type != TYPE_IDEV) {
		479	rc = -EINVAL;
		480	goto out_dev_obj;
		481	}
		482
		483	rc = iommufd_device_replace(sobj->idev.idev, &pt_id);
		484	if (rc)
		485	goto out_dev_obj;
		486
		487	cmd->mock_domain_replace.pt_id = pt_id;
		488	rc = iommufd_ucmd_respond(ucmd, sizeof(*cmd));
		489
		490	out_dev_obj:
		491	iommufd_put_object(dev_obj);
		492	return rc;
		493	}
		494
458	/* Add an additional reserved IOVA to the IOAS */	495	/* Add an additional reserved IOVA to the IOAS */
459	static int iommufd_test_add_reserved(struct iommufd_ucmd *ucmd,	496	static int iommufd_test_add_reserved(struct iommufd_ucmd *ucmd,
460	unsigned int mockpt_id,	497	unsigned int mockpt_id,
@@ -748,6 +785,22 @@ out_free_staccess: · 287 unmodified lines
748	return rc;	785	return rc;
749	}	786	}
750		787
		788	static int iommufd_test_access_replace_ioas(struct iommufd_ucmd *ucmd,
		789	unsigned int access_id,
		790	unsigned int ioas_id)
		791	{
		792	struct selftest_access *staccess;
		793	int rc;
		794
		795	staccess = iommufd_access_get(access_id);
		796	if (IS_ERR(staccess))
		797	return PTR_ERR(staccess);
		798
		799	rc = iommufd_access_replace(staccess->access, ioas_id);
		800	fput(staccess->file);
		801	return rc;
		802	}
		803
751	/* Check that the pages in a page array match the pages in the user VA */	804	/* Check that the pages in a page array match the pages in the user VA */
752	static int iommufd_test_check_pages(void __user uptr, struct page *pages,	805	static int iommufd_test_check_pages(void __user uptr, struct page *pages,
753	size_t npages)	806	size_t npages)
@@ -948,6 +1001,9 @@ int iommufd_test(struct iommufd_ucmd *ucmd) · 194 unmodified lines
948	cmd->add_reserved.length);	1001	cmd->add_reserved.length);
949	case IOMMU_TEST_OP_MOCK_DOMAIN:	1002	case IOMMU_TEST_OP_MOCK_DOMAIN:
950	return iommufd_test_mock_domain(ucmd, cmd);	1003	return iommufd_test_mock_domain(ucmd, cmd);
		1004	case IOMMU_TEST_OP_MOCK_DOMAIN_REPLACE:
		1005	return iommufd_test_mock_domain_replace(
		1006	ucmd, cmd->id, cmd->mock_domain_replace.pt_id, cmd);
951	case IOMMU_TEST_OP_MD_CHECK_MAP:	1007	case IOMMU_TEST_OP_MD_CHECK_MAP:
952	return iommufd_test_md_check_pa(	1008	return iommufd_test_md_check_pa(
953	ucmd, cmd->id, cmd->check_map.iova,	1009	ucmd, cmd->id, cmd->check_map.iova,
@@ -960,6 +1016,9 @@ int iommufd_test(struct iommufd_ucmd *ucmd) · 6 unmodified lines
960	case IOMMU_TEST_OP_CREATE_ACCESS:	1016	case IOMMU_TEST_OP_CREATE_ACCESS:
961	return iommufd_test_create_access(ucmd, cmd->id,	1017	return iommufd_test_create_access(ucmd, cmd->id,
962	cmd->create_access.flags);	1018	cmd->create_access.flags);
		1019	case IOMMU_TEST_OP_ACCESS_REPLACE_IOAS:
		1020	return iommufd_test_access_replace_ioas(
		1021	ucmd, cmd->id, cmd->access_replace_ioas.ioas_id);
963	case IOMMU_TEST_OP_ACCESS_PAGES:	1022	case IOMMU_TEST_OP_ACCESS_PAGES:
964	return iommufd_test_access_pages(	1023	return iommufd_test_access_pages(
965	ucmd, cmd->id, cmd->access_pages.iova,	1024	ucmd, cmd->id, cmd->access_pages.iova,
@@ -992,15 +1051,57 @@ bool iommufd_should_fail(void) · 26 unmodified lines
992	return should_fail(&fail_iommufd, 1);	1051	return should_fail(&fail_iommufd, 1);
993	}	1052	}
994		1053
995	void __init iommufd_test_init(void)	1054	int __init iommufd_test_init(void)
996	{	1055	{
		1056	struct platform_device_info pdevinfo = {
		1057	.name = "iommufd_selftest_iommu",
		1058	};
		1059	int rc;
		1060
997	dbgfs_root =	1061	dbgfs_root =
998	fault_create_debugfs_attr("fail_iommufd", NULL, &fail_iommufd);	1062	fault_create_debugfs_attr("fail_iommufd", NULL, &fail_iommufd);
999	WARN_ON(bus_register(&iommufd_mock_bus_type));	1063
		1064	selftest_iommu_dev = platform_device_register_full(&pdevinfo);
		1065	if (IS_ERR(selftest_iommu_dev)) {
		1066	rc = PTR_ERR(selftest_iommu_dev);
		1067	goto err_dbgfs;
		1068	}
		1069
		1070	rc = bus_register(&iommufd_mock_bus_type.bus);
		1071	if (rc)
		1072	goto err_platform;
		1073
		1074	rc = iommu_device_sysfs_add(&mock_iommu_device,
		1075	&selftest_iommu_dev->dev, NULL, "%s",
		1076	dev_name(&selftest_iommu_dev->dev));
		1077	if (rc)
		1078	goto err_bus;
		1079
		1080	rc = iommu_device_register_bus(&mock_iommu_device, &mock_ops,
		1081	&iommufd_mock_bus_type.bus,
		1082	&iommufd_mock_bus_type.nb);
		1083	if (rc)
		1084	goto err_sysfs;
		1085	return 0;
		1086
		1087	err_sysfs:
		1088	iommu_device_sysfs_remove(&mock_iommu_device);
		1089	err_bus:
		1090	bus_unregister(&iommufd_mock_bus_type.bus);
		1091	err_platform:
		1092	platform_device_unregister(selftest_iommu_dev);
		1093	err_dbgfs:
		1094	debugfs_remove_recursive(dbgfs_root);
		1095	return rc;
1000	}	1096	}
1001		1097
1002	void iommufd_test_exit(void)	1098	void iommufd_test_exit(void)
1003	{	1099	{
		1100	iommu_device_sysfs_remove(&mock_iommu_device);
		1101	iommu_device_unregister_bus(&mock_iommu_device,
		1102	&iommufd_mock_bus_type.bus,
		1103	&iommufd_mock_bus_type.nb);
		1104	bus_unregister(&iommufd_mock_bus_type.bus);
		1105	platform_device_unregister(selftest_iommu_dev);
1004	debugfs_remove_recursive(dbgfs_root);	1106	debugfs_remove_recursive(dbgfs_root);
1005	bus_unregister(&iommufd_mock_bus_type);
1006	}	1107	}

drivers/vfio/iommufd.c+6−5

@@ -146,9 +146,9 @@ int vfio_iommufd_physical_attach_ioas(struct vfio_device vdev, u32 pt_id) · 145 unmodified lines
146	return -EINVAL;	146	return -EINVAL;
147		147
148	if (vdev->iommufd_attached)	148	if (vdev->iommufd_attached)
149	return -EBUSY;	149	rc = iommufd_device_replace(vdev->iommufd_device, pt_id);
150		150	else
151	rc = iommufd_device_attach(vdev->iommufd_device, pt_id);	151	rc = iommufd_device_attach(vdev->iommufd_device, pt_id);
152	if (rc)	152	if (rc)
153	return rc;	153	return rc;
154	vdev->iommufd_attached = true;	154	vdev->iommufd_attached = true;
@@ -223,8 +223,9 @@ int vfio_iommufd_emulated_attach_ioas(struct vfio_device vdev, u32 pt_id) · 68 unmodified lines
223	lockdep_assert_held(&vdev->dev_set->lock);	223	lockdep_assert_held(&vdev->dev_set->lock);
224		224
225	if (vdev->iommufd_attached)	225	if (vdev->iommufd_attached)
226	return -EBUSY;	226	rc = iommufd_access_replace(vdev->iommufd_access, *pt_id);
227	rc = iommufd_access_attach(vdev->iommufd_access, *pt_id);	227	else
		228	rc = iommufd_access_attach(vdev->iommufd_access, *pt_id);
228	if (rc)	229	if (rc)
229	return rc;	230	return rc;
230	vdev->iommufd_attached = true;	231	vdev->iommufd_attached = true;

drivers/vfio/vfio_main.c+4−0

 	/* group->container cannot change while a vfio device is open */
 	if (!pages || !npage || WARN_ON(!vfio_assert_device_open(device)))
 		return -EINVAL;
+	if (!device->ops->dma_unmap)
+		return -EINVAL;
 	if (vfio_device_has_container(device))
 		return vfio_device_container_pin_pages(device, iova,
 						       npage, prot, pages);
 {
 	if (WARN_ON(!vfio_assert_device_open(device)))
 		return;
+	if (WARN_ON(!device->ops->dma_unmap))
+		return;
 	if (vfio_device_has_container(device)) {
 		vfio_device_container_unpin_pages(device, iova, npage);

include/linux/iommu.h+5−11

 /**
  * struct iommu_ops - iommu ops and capabilities
  * @capable: check capability
+ * @hw_info: report iommu hardware information. The data buffer returned by this
+ *           op is allocated in the iommu driver and freed by the caller after
+ *           use. The information type is one of enum iommu_hw_info_type defined
+ *           in include/uapi/linux/iommufd.h.
  * @domain_alloc: allocate iommu domain
  * @probe_device: Add device to iommu driver handling
  * @release_device: Remove device from iommu driver handling
  */
 struct iommu_ops {
 	bool (*capable)(struct device *dev, enum iommu_cap);
+	void *(*hw_info)(struct device *dev, u32 *length, u32 *type);
 	/* Domain allocation and freeing by the iommu driver */
 	struct iommu_domain *(*domain_alloc)(unsigned iommu_domain_type);
 	};
 }
-static inline const struct iommu_ops *dev_iommu_ops(struct device *dev)
-{
-	/*
-	 * Assume that valid ops must be installed if iommu_probe_device()
-	 * has succeeded. The device ops are essentially for internal use
-	 * within the IOMMU subsystem itself, so we should be able to trust
-	 * ourselves not to misuse the helper.
-	 */
-	return dev->iommu->iommu_dev->ops;
-}
 extern int bus_iommu_probe(const struct bus_type *bus);
 extern bool iommu_present(const struct bus_type *bus);
 extern bool device_iommu_capable(struct device *dev, enum iommu_cap cap);

include/linux/iommufd.h+2−0

 void iommufd_device_unbind(struct iommufd_device *idev);
 int iommufd_device_attach(struct iommufd_device *idev, u32 *pt_id);
+int iommufd_device_replace(struct iommufd_device *idev, u32 *pt_id);
 void iommufd_device_detach(struct iommufd_device *idev);
 struct iommufd_ctx *iommufd_device_to_ictx(struct iommufd_device *idev);
 		      const struct iommufd_access_ops *ops, void *data, u32 *id);
 void iommufd_access_destroy(struct iommufd_access *access);
 int iommufd_access_attach(struct iommufd_access *access, u32 ioas_id);
+int iommufd_access_replace(struct iommufd_access *access, u32 ioas_id);
 void iommufd_access_detach(struct iommufd_access *access);
 void iommufd_ctx_get(struct iommufd_ctx *ictx);

include/uapi/linux/iommufd.h+97−0

@@ -45,6 +45,8 @@ enum { · 44 unmodified lines
45	IOMMUFD_CMD_IOAS_UNMAP,	45	IOMMUFD_CMD_IOAS_UNMAP,
46	IOMMUFD_CMD_OPTION,	46	IOMMUFD_CMD_OPTION,
47	IOMMUFD_CMD_VFIO_IOAS,	47	IOMMUFD_CMD_VFIO_IOAS,
		48	IOMMUFD_CMD_HWPT_ALLOC,
		49	IOMMUFD_CMD_GET_HW_INFO,
48	};	50	};
49		51
50	/**	52	/**
@@ -344,4 +346,99 @@ struct iommu_vfio_ioas { · 293 unmodified lines
344	__u16 __reserved;	346	__u16 __reserved;
345	};	347	};
346	#define IOMMU_VFIO_IOAS _IO(IOMMUFD_TYPE, IOMMUFD_CMD_VFIO_IOAS)	348	#define IOMMU_VFIO_IOAS _IO(IOMMUFD_TYPE, IOMMUFD_CMD_VFIO_IOAS)
		349
		350	/**
		351	* struct iommu_hwpt_alloc - ioctl(IOMMU_HWPT_ALLOC)
		352	* @size: sizeof(struct iommu_hwpt_alloc)
		353	* @flags: Must be 0
		354	* @dev_id: The device to allocate this HWPT for
		355	* @pt_id: The IOAS to connect this HWPT to
		356	* @out_hwpt_id: The ID of the new HWPT
		357	* @__reserved: Must be 0
		358	*
		359	* Explicitly allocate a hardware page table object. This is the same object
		360	* type that is returned by iommufd_device_attach() and represents the
		361	* underlying iommu driver's iommu_domain kernel object.
		362	*
		363	* A HWPT will be created with the IOVA mappings from the given IOAS.
		364	*/
		365	struct iommu_hwpt_alloc {
		366	__u32 size;
		367	__u32 flags;
		368	__u32 dev_id;
		369	__u32 pt_id;
		370	__u32 out_hwpt_id;
		371	__u32 __reserved;
		372	};
		373	#define IOMMU_HWPT_ALLOC _IO(IOMMUFD_TYPE, IOMMUFD_CMD_HWPT_ALLOC)
		374
		375	/**
		376	* struct iommu_hw_info_vtd - Intel VT-d hardware information
		377	*
		378	* @flags: Must be 0
		379	* @__reserved: Must be 0
		380	*
		381	* @cap_reg: Value of Intel VT-d capability register defined in VT-d spec
		382	* section 11.4.2 Capability Register.
		383	* @ecap_reg: Value of Intel VT-d capability register defined in VT-d spec
		384	* section 11.4.3 Extended Capability Register.
		385	*
		386	* User needs to understand the Intel VT-d specification to decode the
		387	* register value.
		388	*/
		389	struct iommu_hw_info_vtd {
		390	__u32 flags;
		391	__u32 __reserved;
		392	__aligned_u64 cap_reg;
		393	__aligned_u64 ecap_reg;
		394	};
		395
		396	/**
		397	* enum iommu_hw_info_type - IOMMU Hardware Info Types
		398	* @IOMMU_HW_INFO_TYPE_NONE: Used by the drivers that do not report hardware
		399	* info
		400	* @IOMMU_HW_INFO_TYPE_INTEL_VTD: Intel VT-d iommu info type
		401	*/
		402	enum iommu_hw_info_type {
		403	IOMMU_HW_INFO_TYPE_NONE,
		404	IOMMU_HW_INFO_TYPE_INTEL_VTD,
		405	};
		406
		407	/**
		408	* struct iommu_hw_info - ioctl(IOMMU_GET_HW_INFO)
		409	* @size: sizeof(struct iommu_hw_info)
		410	* @flags: Must be 0
		411	* @dev_id: The device bound to the iommufd
		412	* @data_len: Input the length of a user buffer in bytes. Output the length of
		413	* data that kernel supports
		414	* @data_uptr: User pointer to a user-space buffer used by the kernel to fill
		415	* the iommu type specific hardware information data
		416	* @out_data_type: Output the iommu hardware info type as defined in the enum
		417	* iommu_hw_info_type.
		418	* @__reserved: Must be 0
		419	*
		420	* Query an iommu type specific hardware information data from an iommu behind
		421	* a given device that has been bound to iommufd. This hardware info data will
		422	* be used to sync capabilities between the virtual iommu and the physical
		423	* iommu, e.g. a nested translation setup needs to check the hardware info, so
		424	* a guest stage-1 page table can be compatible with the physical iommu.
		425	*
		426	* To capture an iommu type specific hardware information data, @data_uptr and
		427	* its length @data_len must be provided. Trailing bytes will be zeroed if the
		428	* user buffer is larger than the data that kernel has. Otherwise, kernel only
		429	* fills the buffer using the given length in @data_len. If the ioctl succeeds,
		430	* @data_len will be updated to the length that kernel actually supports,
		431	* @out_data_type will be filled to decode the data filled in the buffer
		432	* pointed by @data_uptr. Input @data_len == zero is allowed.
		433	*/
		434	struct iommu_hw_info {
		435	__u32 size;
		436	__u32 flags;
		437	__u32 dev_id;
		438	__u32 data_len;
		439	__aligned_u64 data_uptr;
		440	__u32 out_data_type;
		441	__u32 __reserved;
		442	};
		443	#define IOMMU_GET_HW_INFO _IO(IOMMUFD_TYPE, IOMMUFD_CMD_GET_HW_INFO)
347	#endif	444	#endif

include/uapi/linux/vfio.h+6−0

  * Undo by VFIO_DEVICE_DETACH_IOMMUFD_PT or device fd close.  This is only
  * allowed on cdev fds.
  *
+ * If a vfio device is currently attached to a valid hw_pagetable, without doing
+ * a VFIO_DEVICE_DETACH_IOMMUFD_PT, a second VFIO_DEVICE_ATTACH_IOMMUFD_PT ioctl
+ * passing in another hw_pagetable (hwpt) id is allowed. This action, also known
+ * as a hw_pagetable replacement, will replace the device's currently attached
+ * hw_pagetable with a new hw_pagetable corresponding to the given pt_id.
+ *
  * Return: 0 on success, -errno on failure.
  */
 struct vfio_device_attach_iommufd_pt {

tools/testing/selftests/iommu/iommufd.c+118−12

 #include "iommufd_utils.h"
-static void *buffer;
-static unsigned long PAGE_SIZE;
 static unsigned long HUGEPAGE_SIZE;
 #define MOCK_PAGE_SIZE (PAGE_SIZE / 2)
 	}
 	TEST_LENGTH(iommu_destroy, IOMMU_DESTROY);
+	TEST_LENGTH(iommu_hw_info, IOMMU_GET_HW_INFO);
 	TEST_LENGTH(iommu_ioas_alloc, IOMMU_IOAS_ALLOC);
 	TEST_LENGTH(iommu_ioas_iova_ranges, IOMMU_IOAS_IOVA_RANGES);
 	TEST_LENGTH(iommu_ioas_allow_iovas, IOMMU_IOAS_ALLOW_IOVAS);
 	uint32_t ioas_id;
 	uint32_t stdev_id;
 	uint32_t hwpt_id;
+	uint32_t device_id;
 	uint64_t base_iova;
 };
 	for (i = 0; i != variant->mock_domains; i++) {
 		test_cmd_mock_domain(self->ioas_id, &self->stdev_id,
-				     &self->hwpt_id);
+				     &self->hwpt_id, &self->device_id);
 		self->base_iova = MOCK_APERTURE_START;
 	}
 }
 {
 	/* Create a device attached directly to a hwpt */
 	if (self->stdev_id) {
-		test_cmd_mock_domain(self->hwpt_id, NULL, NULL);
+		test_cmd_mock_domain(self->hwpt_id, NULL, NULL, NULL);
 	} else {
 		test_err_mock_domain(ENOENT, self->hwpt_id, NULL, NULL);
 	}
 	}
 }
+TEST_F(iommufd_ioas, get_hw_info)
+{
+	struct iommu_test_hw_info buffer_exact;
+	struct iommu_test_hw_info_buffer_larger {
+		struct iommu_test_hw_info info;
+		uint64_t trailing_bytes;
+	} buffer_larger;
+	struct iommu_test_hw_info_buffer_smaller {
+		__u32 flags;
+	} buffer_smaller;
+	if (self->device_id) {
+		/* Provide a zero-size user_buffer */
+		test_cmd_get_hw_info(self->device_id, NULL, 0);
+		/* Provide a user_buffer with exact size */
+		test_cmd_get_hw_info(self->device_id, &buffer_exact, sizeof(buffer_exact));
+		/*
+		 * Provide a user_buffer with size larger than the exact size to check if
+		 * kernel zero the trailing bytes.
+		 */
+		test_cmd_get_hw_info(self->device_id, &buffer_larger, sizeof(buffer_larger));
+		/*
+		 * Provide a user_buffer with size smaller than the exact size to check if
+		 * the fields within the size range still gets updated.
+		 */
+		test_cmd_get_hw_info(self->device_id, &buffer_smaller, sizeof(buffer_smaller));
+	} else {
+		test_err_get_hw_info(ENOENT, self->device_id,
+				     &buffer_exact, sizeof(buffer_exact));
+		test_err_get_hw_info(ENOENT, self->device_id,
+				     &buffer_larger, sizeof(buffer_larger));
+	}
+}
 TEST_F(iommufd_ioas, area)
 {
 	int i;
 				   _IOMMU_TEST_CMD(IOMMU_TEST_OP_ACCESS_PAGES),
 				   &access_cmd));
 		test_cmd_mock_domain(self->ioas_id, &mock_stdev_id,
-				     &mock_hwpt_id);
+				     &mock_hwpt_id, NULL);
 		check_map_cmd.id = mock_hwpt_id;
 		ASSERT_EQ(0, ioctl(self->fd,
 				   _IOMMU_TEST_CMD(IOMMU_TEST_OP_MD_CHECK_MAP),
 		 * If a domain already existed then everything was pinned within
 		 * the fork, so this copies from one domain to another.
 		 */
-		test_cmd_mock_domain(self->ioas_id, NULL, NULL);
+		test_cmd_mock_domain(self->ioas_id, NULL, NULL, NULL);
 		check_access_rw(_metadata, self->fd, access_id,
 				MOCK_APERTURE_START, 0);
 	ASSERT_EQ(8, read(efd, &tmp, sizeof(tmp)));
 	/* Read pages from the remote process */
-	test_cmd_mock_domain(self->ioas_id, NULL, NULL);
+	test_cmd_mock_domain(self->ioas_id, NULL, NULL, NULL);
 	check_access_rw(_metadata, self->fd, access_id, MOCK_APERTURE_START, 0);
 	ASSERT_EQ(0, close(pipefds[1]));
 	uint32_t ioas_id;
 	uint32_t hwpt_id;
 	uint32_t hwpt_ids[2];
+	uint32_t stdev_ids[2];
+	uint32_t idev_ids[2];
 	int mmap_flags;
 	size_t mmap_buf_size;
 };
 	ASSERT_GE(ARRAY_SIZE(self->hwpt_ids), variant->mock_domains);
 	for (i = 0; i != variant->mock_domains; i++)
-		test_cmd_mock_domain(self->ioas_id, NULL, &self->hwpt_ids[i]);
+		test_cmd_mock_domain(self->ioas_id, &self->stdev_ids[i],
+				     &self->hwpt_ids[i], &self->idev_ids[i]);
 	self->hwpt_id = self->hwpt_ids[0];
 	self->mmap_flags = MAP_SHARED | MAP_ANONYMOUS;
 			/* Add and destroy a domain while the area exists */
 			old_id = self->hwpt_ids[1];
 			test_cmd_mock_domain(self->ioas_id, &mock_stdev_id,
-					     &self->hwpt_ids[1]);
+					     &self->hwpt_ids[1], NULL);
 			check_mock_iova(buf + start, iova, length);
 			check_refs(buf + start / PAGE_SIZE * PAGE_SIZE,
 		.dst_iova = MOCK_APERTURE_START,
 		.length = BUFFER_SIZE,
 	};
-	unsigned int ioas_id;
+	struct iommu_ioas_unmap unmap_cmd = {
+		.size = sizeof(unmap_cmd),
+		.ioas_id = self->ioas_id,
+		.iova = MOCK_APERTURE_START,
+		.length = BUFFER_SIZE,
+	};
+	unsigned int new_ioas_id, ioas_id;
 	/* Pin the pages in an IOAS with no domains then copy to an IOAS with domains */
 	test_ioctl_ioas_alloc(&ioas_id);
 	ASSERT_EQ(0, ioctl(self->fd, IOMMU_IOAS_COPY, &copy_cmd));
 	check_mock_iova(buffer, MOCK_APERTURE_START, BUFFER_SIZE);
+	/* Now replace the ioas with a new one */
+	test_ioctl_ioas_alloc(&new_ioas_id);
+	test_ioctl_ioas_map_id(new_ioas_id, buffer, BUFFER_SIZE,
+			       &copy_cmd.src_iova);
+	test_cmd_access_replace_ioas(access_cmd.id, new_ioas_id);
+	/* Destroy the old ioas and cleanup copied mapping */
+	ASSERT_EQ(0, ioctl(self->fd, IOMMU_IOAS_UNMAP, &unmap_cmd));
+	test_ioctl_destroy(ioas_id);
+	/* Then run the same test again with the new ioas */
+	access_cmd.access_pages.iova = copy_cmd.src_iova;
+	ASSERT_EQ(0,
+		  ioctl(self->fd, _IOMMU_TEST_CMD(IOMMU_TEST_OP_ACCESS_PAGES),
+			&access_cmd));
+	copy_cmd.src_ioas_id = new_ioas_id;
+	ASSERT_EQ(0, ioctl(self->fd, IOMMU_IOAS_COPY, &copy_cmd));
+	check_mock_iova(buffer, MOCK_APERTURE_START, BUFFER_SIZE);
 	test_cmd_destroy_access_pages(
 		access_cmd.id, access_cmd.access_pages.out_access_pages_id);
 	test_cmd_destroy_access(access_cmd.id);
+	test_ioctl_destroy(new_ioas_id);
+}
+TEST_F(iommufd_mock_domain, replace)
+{
+	uint32_t ioas_id;
+	test_ioctl_ioas_alloc(&ioas_id);
+	test_cmd_mock_domain_replace(self->stdev_ids[0], ioas_id);
+	/*
+	 * Replacing the IOAS causes the prior HWPT to be deallocated, thus we
+	 * should get enoent when we try to use it.
+	 */
+	if (variant->mock_domains == 1)
+		test_err_mock_domain_replace(ENOENT, self->stdev_ids[0],
+					     self->hwpt_ids[0]);
+	test_cmd_mock_domain_replace(self->stdev_ids[0], ioas_id);
+	if (variant->mock_domains >= 2) {
+		test_cmd_mock_domain_replace(self->stdev_ids[0],
+					     self->hwpt_ids[1]);
+		test_cmd_mock_domain_replace(self->stdev_ids[0],
+					     self->hwpt_ids[1]);
+		test_cmd_mock_domain_replace(self->stdev_ids[0],
+					     self->hwpt_ids[0]);
+	}
+	test_cmd_mock_domain_replace(self->stdev_ids[0], self->ioas_id);
 	test_ioctl_destroy(ioas_id);
 }
+TEST_F(iommufd_mock_domain, alloc_hwpt)
+{
+	int i;
+	for (i = 0; i != variant->mock_domains; i++) {
+		uint32_t stddev_id;
+		uint32_t hwpt_id;
+		test_cmd_hwpt_alloc(self->idev_ids[0], self->ioas_id, &hwpt_id);
+		test_cmd_mock_domain(hwpt_id, &stddev_id, NULL, NULL);
+		test_ioctl_destroy(stddev_id);
+		test_ioctl_destroy(hwpt_id);
+	}
+}
 /* VFIO compatibility IOCTLs */
 TEST_F(iommufd, simple_ioctls)
 	/* Create what VFIO would consider a group */
 	test_ioctl_ioas_alloc(&self->ioas_id);
-	test_cmd_mock_domain(self->ioas_id, NULL, NULL);
+	test_cmd_mock_domain(self->ioas_id, NULL, NULL, NULL);
 	/* Attach it to the vfio compat */
 	vfio_ioas_cmd.ioas_id = self->ioas_id;

tools/testing/selftests/iommu/iommufd_fail_nth.c+64−7

@@ -41,6 +41,8 @@ static int writeat(int dfd, const char fn, const char val) · 40 unmodified lines
41		41
42	static __attribute__((constructor)) void setup_buffer(void)	42	static __attribute__((constructor)) void setup_buffer(void)
43	{	43	{
		44	PAGE_SIZE = sysconf(_SC_PAGE_SIZE);
		45
44	BUFFER_SIZE = 210241024;	46	BUFFER_SIZE = 210241024;
45		47
46	buffer = mmap(0, BUFFER_SIZE, PROT_READ \| PROT_WRITE,	48	buffer = mmap(0, BUFFER_SIZE, PROT_READ \| PROT_WRITE,
@@ -313,7 +315,7 @@ TEST_FAIL_NTH(basic_fail_nth, map_domain) · 266 unmodified lines
313		315
314	fail_nth_enable();	316	fail_nth_enable();
315		317
316	if (_test_cmd_mock_domain(self->fd, ioas_id, &stdev_id, &hwpt_id))	318	if (_test_cmd_mock_domain(self->fd, ioas_id, &stdev_id, &hwpt_id, NULL))
317	return -1;	319	return -1;
318		320
319	if (_test_ioctl_ioas_map(self->fd, ioas_id, buffer, 262144, &iova,	321	if (_test_ioctl_ioas_map(self->fd, ioas_id, buffer, 262144, &iova,
@@ -324,7 +326,7 @@ TEST_FAIL_NTH(basic_fail_nth, map_domain) · 4 unmodified lines
324	if (_test_ioctl_destroy(self->fd, stdev_id))	326	if (_test_ioctl_destroy(self->fd, stdev_id))
325	return -1;	327	return -1;
326		328
327	if (_test_cmd_mock_domain(self->fd, ioas_id, &stdev_id, &hwpt_id))	329	if (_test_cmd_mock_domain(self->fd, ioas_id, &stdev_id, &hwpt_id, NULL))
328	return -1;	330	return -1;
329	return 0;	331	return 0;
330	}	332	}
@@ -348,12 +350,13 @@ TEST_FAIL_NTH(basic_fail_nth, map_two_domains) · 17 unmodified lines
348	if (_test_ioctl_set_temp_memory_limit(self->fd, 32))	350	if (_test_ioctl_set_temp_memory_limit(self->fd, 32))
349	return -1;	351	return -1;
350		352
351	if (_test_cmd_mock_domain(self->fd, ioas_id, &stdev_id, &hwpt_id))	353	if (_test_cmd_mock_domain(self->fd, ioas_id, &stdev_id, &hwpt_id, NULL))
352	return -1;	354	return -1;
353		355
354	fail_nth_enable();	356	fail_nth_enable();
355		357
356	if (_test_cmd_mock_domain(self->fd, ioas_id, &stdev_id2, &hwpt_id2))	358	if (_test_cmd_mock_domain(self->fd, ioas_id, &stdev_id2, &hwpt_id2,
		359	NULL))
357	return -1;	360	return -1;
358		361
359	if (_test_ioctl_ioas_map(self->fd, ioas_id, buffer, 262144, &iova,	362	if (_test_ioctl_ioas_map(self->fd, ioas_id, buffer, 262144, &iova,
@@ -367,9 +370,10 @@ TEST_FAIL_NTH(basic_fail_nth, map_two_domains) · 7 unmodified lines
367	if (_test_ioctl_destroy(self->fd, stdev_id2))	370	if (_test_ioctl_destroy(self->fd, stdev_id2))
368	return -1;	371	return -1;
369		372
370	if (_test_cmd_mock_domain(self->fd, ioas_id, &stdev_id, &hwpt_id))	373	if (_test_cmd_mock_domain(self->fd, ioas_id, &stdev_id, &hwpt_id, NULL))
371	return -1;	374	return -1;
372	if (_test_cmd_mock_domain(self->fd, ioas_id, &stdev_id2, &hwpt_id2))	375	if (_test_cmd_mock_domain(self->fd, ioas_id, &stdev_id2, &hwpt_id2,
		376	NULL))
373	return -1;	377	return -1;
374	return 0;	378	return 0;
375	}	379	}
@@ -526,7 +530,7 @@ TEST_FAIL_NTH(basic_fail_nth, access_pin_domain) · 150 unmodified lines
526	if (_test_ioctl_set_temp_memory_limit(self->fd, 32))	530	if (_test_ioctl_set_temp_memory_limit(self->fd, 32))
527	return -1;	531	return -1;
528		532
529	if (_test_cmd_mock_domain(self->fd, ioas_id, &stdev_id, &hwpt_id))	533	if (_test_cmd_mock_domain(self->fd, ioas_id, &stdev_id, &hwpt_id, NULL))
530	return -1;	534	return -1;
531		535
532	if (_test_ioctl_ioas_map(self->fd, ioas_id, buffer, BUFFER_SIZE, &iova,	536	if (_test_ioctl_ioas_map(self->fd, ioas_id, buffer, BUFFER_SIZE, &iova,
@@ -569,4 +573,57 @@ TEST_FAIL_NTH(basic_fail_nth, access_pin_domain) · 36 unmodified lines
569	return 0;	573	return 0;
570	}	574	}
571		575
		576	/* device.c */
		577	TEST_FAIL_NTH(basic_fail_nth, device)
		578	{
		579	struct iommu_test_hw_info info;
		580	uint32_t ioas_id;
		581	uint32_t ioas_id2;
		582	uint32_t stdev_id;
		583	uint32_t idev_id;
		584	uint32_t hwpt_id;
		585	__u64 iova;
		586
		587	self->fd = open("/dev/iommu", O_RDWR);
		588	if (self->fd == -1)
		589	return -1;
		590
		591	if (_test_ioctl_ioas_alloc(self->fd, &ioas_id))
		592	return -1;
		593
		594	if (_test_ioctl_ioas_alloc(self->fd, &ioas_id2))
		595	return -1;
		596
		597	iova = MOCK_APERTURE_START;
		598	if (_test_ioctl_ioas_map(self->fd, ioas_id, buffer, PAGE_SIZE, &iova,
		599	IOMMU_IOAS_MAP_FIXED_IOVA \|
		600	IOMMU_IOAS_MAP_WRITEABLE \|
		601	IOMMU_IOAS_MAP_READABLE))
		602	return -1;
		603	if (_test_ioctl_ioas_map(self->fd, ioas_id2, buffer, PAGE_SIZE, &iova,
		604	IOMMU_IOAS_MAP_FIXED_IOVA \|
		605	IOMMU_IOAS_MAP_WRITEABLE \|
		606	IOMMU_IOAS_MAP_READABLE))
		607	return -1;
		608
		609	fail_nth_enable();
		610
		611	if (_test_cmd_mock_domain(self->fd, ioas_id, &stdev_id, NULL,
		612	&idev_id))
		613	return -1;
		614
		615	if (_test_cmd_get_hw_info(self->fd, idev_id, &info, sizeof(info)))
		616	return -1;
		617
		618	if (_test_cmd_hwpt_alloc(self->fd, idev_id, ioas_id, &hwpt_id))
		619	return -1;
		620
		621	if (_test_cmd_mock_domain_replace(self->fd, stdev_id, ioas_id2, NULL))
		622	return -1;
		623
		624	if (_test_cmd_mock_domain_replace(self->fd, stdev_id, hwpt_id, NULL))
		625	return -1;
		626	return 0;
		627	}
		628
572	TEST_HARNESS_MAIN	629	TEST_HARNESS_MAIN

tools/testing/selftests/iommu/iommufd_utils.h+139−5

@@ -19,6 +19,12 @@ · 18 unmodified lines
19	static void *buffer;	19	static void *buffer;
20	static unsigned long BUFFER_SIZE;	20	static unsigned long BUFFER_SIZE;
21		21
		22	static unsigned long PAGE_SIZE;
		23
		24	#define sizeof_field(TYPE, MEMBER) sizeof((((TYPE *)0)->MEMBER))
		25	#define offsetofend(TYPE, MEMBER) \
		26	(offsetof(TYPE, MEMBER) + sizeof_field(TYPE, MEMBER))
		27
22	/*	28	/*
23	* Have the kernel check the refcount on pages. I don't know why a freshly	29	* Have the kernel check the refcount on pages. I don't know why a freshly
24	* mmap'd anon non-compound page starts out with a ref of 3	30	* mmap'd anon non-compound page starts out with a ref of 3
@@ -39,7 +45,7 @@ static unsigned long BUFFER_SIZE; · 14 unmodified lines
39	})	45	})
40		46
41	static int _test_cmd_mock_domain(int fd, unsigned int ioas_id, __u32 *stdev_id,	47	static int _test_cmd_mock_domain(int fd, unsigned int ioas_id, __u32 *stdev_id,
42	__u32 *hwpt_id)	48	__u32 hwpt_id, __u32 idev_id)
43	{	49	{
44	struct iommu_test_cmd cmd = {	50	struct iommu_test_cmd cmd = {
45	.size = sizeof(cmd),	51	.size = sizeof(cmd),
@@ -57,14 +63,84 @@ static int _test_cmd_mock_domain(int fd, unsigned int ioas_id, __u32 *stdev_id, · 11 unmodified lines
57	assert(cmd.id != 0);	63	assert(cmd.id != 0);
58	if (hwpt_id)	64	if (hwpt_id)
59	*hwpt_id = cmd.mock_domain.out_hwpt_id;	65	*hwpt_id = cmd.mock_domain.out_hwpt_id;
		66	if (idev_id)
		67	*idev_id = cmd.mock_domain.out_idev_id;
60	return 0;	68	return 0;
61	}	69	}
62	#define test_cmd_mock_domain(ioas_id, stdev_id, hwpt_id) \	70	#define test_cmd_mock_domain(ioas_id, stdev_id, hwpt_id, idev_id) \
63	ASSERT_EQ(0, \	71	ASSERT_EQ(0, _test_cmd_mock_domain(self->fd, ioas_id, stdev_id, \
64	_test_cmd_mock_domain(self->fd, ioas_id, stdev_id, hwpt_id))	72	hwpt_id, idev_id))
65	#define test_err_mock_domain(_errno, ioas_id, stdev_id, hwpt_id) \	73	#define test_err_mock_domain(_errno, ioas_id, stdev_id, hwpt_id) \
66	EXPECT_ERRNO(_errno, _test_cmd_mock_domain(self->fd, ioas_id, \	74	EXPECT_ERRNO(_errno, _test_cmd_mock_domain(self->fd, ioas_id, \
67	stdev_id, hwpt_id))	75	stdev_id, hwpt_id, NULL))
		76
		77	static int _test_cmd_mock_domain_replace(int fd, __u32 stdev_id, __u32 pt_id,
		78	__u32 *hwpt_id)
		79	{
		80	struct iommu_test_cmd cmd = {
		81	.size = sizeof(cmd),
		82	.op = IOMMU_TEST_OP_MOCK_DOMAIN_REPLACE,
		83	.id = stdev_id,
		84	.mock_domain_replace = {
		85	.pt_id = pt_id,
		86	},
		87	};
		88	int ret;
		89
		90	ret = ioctl(fd, IOMMU_TEST_CMD, &cmd);
		91	if (ret)
		92	return ret;
		93	if (hwpt_id)
		94	*hwpt_id = cmd.mock_domain_replace.pt_id;
		95	return 0;
		96	}
		97
		98	#define test_cmd_mock_domain_replace(stdev_id, pt_id) \
		99	ASSERT_EQ(0, _test_cmd_mock_domain_replace(self->fd, stdev_id, pt_id, \
		100	NULL))
		101	#define test_err_mock_domain_replace(_errno, stdev_id, pt_id) \
		102	EXPECT_ERRNO(_errno, _test_cmd_mock_domain_replace(self->fd, stdev_id, \
		103	pt_id, NULL))
		104
		105	static int _test_cmd_hwpt_alloc(int fd, __u32 device_id, __u32 pt_id,
		106	__u32 *hwpt_id)
		107	{
		108	struct iommu_hwpt_alloc cmd = {
		109	.size = sizeof(cmd),
		110	.dev_id = device_id,
		111	.pt_id = pt_id,
		112	};
		113	int ret;
		114
		115	ret = ioctl(fd, IOMMU_HWPT_ALLOC, &cmd);
		116	if (ret)
		117	return ret;
		118	if (hwpt_id)
		119	*hwpt_id = cmd.out_hwpt_id;
		120	return 0;
		121	}
		122
		123	#define test_cmd_hwpt_alloc(device_id, pt_id, hwpt_id) \
		124	ASSERT_EQ(0, _test_cmd_hwpt_alloc(self->fd, device_id, pt_id, hwpt_id))
		125
		126	static int _test_cmd_access_replace_ioas(int fd, __u32 access_id,
		127	unsigned int ioas_id)
		128	{
		129	struct iommu_test_cmd cmd = {
		130	.size = sizeof(cmd),
		131	.op = IOMMU_TEST_OP_ACCESS_REPLACE_IOAS,
		132	.id = access_id,
		133	.access_replace_ioas = { .ioas_id = ioas_id },
		134	};
		135	int ret;
		136
		137	ret = ioctl(fd, IOMMU_TEST_CMD, &cmd);
		138	if (ret)
		139	return ret;
		140	return 0;
		141	}
		142	#define test_cmd_access_replace_ioas(access_id, ioas_id) \
		143	ASSERT_EQ(0, _test_cmd_access_replace_ioas(self->fd, access_id, ioas_id))
68		144
69	static int _test_cmd_create_access(int fd, unsigned int ioas_id,	145	static int _test_cmd_create_access(int fd, unsigned int ioas_id,
70	__u32 *access_id, unsigned int flags)	146	__u32 *access_id, unsigned int flags)
@@ -276,3 +352,61 @@ static void teardown_iommufd(int fd, struct __test_metadata *_metadata) · 205 unmodified lines
276	})	352	})
277		353
278	#endif	354	#endif
		355
		356	/* @data can be NULL */
		357	static int _test_cmd_get_hw_info(int fd, __u32 device_id,
		358	void *data, size_t data_len)
		359	{
		360	struct iommu_test_hw_info info = (struct iommu_test_hw_info )data;
		361	struct iommu_hw_info cmd = {
		362	.size = sizeof(cmd),
		363	.dev_id = device_id,
		364	.data_len = data_len,
		365	.data_uptr = (uint64_t)data,
		366	};
		367	int ret;
		368
		369	ret = ioctl(fd, IOMMU_GET_HW_INFO, &cmd);
		370	if (ret)
		371	return ret;
		372
		373	assert(cmd.out_data_type == IOMMU_HW_INFO_TYPE_SELFTEST);
		374
		375	/*
		376	* The struct iommu_test_hw_info should be the one defined
		377	* by the current kernel.
		378	*/
		379	assert(cmd.data_len == sizeof(struct iommu_test_hw_info));
		380
		381	/*
		382	* Trailing bytes should be 0 if user buffer is larger than
		383	* the data that kernel reports.
		384	*/
		385	if (data_len > cmd.data_len) {
		386	char ptr = (char )(data + cmd.data_len);
		387	int idx = 0;
		388
		389	while (idx < data_len - cmd.data_len) {
		390	assert(!*(ptr + idx));
		391	idx++;
		392	}
		393	}
		394
		395	if (info) {
		396	if (data_len >= offsetofend(struct iommu_test_hw_info, test_reg))
		397	assert(info->test_reg == IOMMU_HW_INFO_SELFTEST_REGVAL);
		398	if (data_len >= offsetofend(struct iommu_test_hw_info, flags))
		399	assert(!info->flags);
		400	}
		401
		402	return 0;
		403	}
		404
		405	#define test_cmd_get_hw_info(device_id, data, data_len) \
		406	ASSERT_EQ(0, _test_cmd_get_hw_info(self->fd, device_id, \
		407	data, data_len))
		408
		409	#define test_err_get_hw_info(_errno, device_id, data, data_len) \
		410	EXPECT_ERRNO(_errno, \
		411	_test_cmd_get_hw_info(self->fd, device_id, \
		412	data, data_len))