lael-labs/linux

drivers/dma/idxd/init.c+1−1

@@ -584,7 +584,7 @@ static int idxd_enable_system_pasid(struct idxd_device *idxd) · 583 unmodified lines
584	* DMA domain is owned by the driver, it should support all valid	584	* DMA domain is owned by the driver, it should support all valid
585	* types such as DMA-FQ, identity, etc.	585	* types such as DMA-FQ, identity, etc.
586	*/	586	*/
587	ret = iommu_attach_device_pasid(domain, dev, pasid);	587	ret = iommu_attach_device_pasid(domain, dev, pasid, NULL);
588	if (ret) {	588	if (ret) {
589	dev_err(dev, "failed to attach device pasid %d, domain type %d",	589	dev_err(dev, "failed to attach device pasid %d, domain type %d",
590	pasid, domain->type);	590	pasid, domain->type);

drivers/iommu/io-pgfault.c+36−27

 }
 EXPORT_SYMBOL_GPL(iopf_free_group);
-static struct iommu_domain *get_domain_for_iopf(struct device *dev,
-						struct iommu_fault *fault)
-{
-	struct iommu_domain *domain;
-	if (fault->prm.flags & IOMMU_FAULT_PAGE_REQUEST_PASID_VALID) {
-		domain = iommu_get_domain_for_dev_pasid(dev, fault->prm.pasid, 0);
-		if (IS_ERR(domain))
-			domain = NULL;
-	} else {
-		domain = iommu_get_domain_for_dev(dev);
-	}
-	if (!domain || !domain->iopf_handler) {
-		dev_warn_ratelimited(dev,
-			"iopf (pasid %d) without domain attached or handler installed\n",
-			 fault->prm.pasid);
-		return NULL;
-	}
-	return domain;
-}
 /* Non-last request of a group. Postpone until the last one. */
 static int report_partial_fault(struct iommu_fault_param *fault_param,
 				struct iommu_fault *fault)
 	list_add(&group->pending_node, &iopf_param->faults);
 	mutex_unlock(&iopf_param->lock);
+	group->fault_count = list_count_nodes(&group->faults);
 	return group;
 }
 	if (group == &abort_group)
 		goto err_abort;
-	group->domain = get_domain_for_iopf(dev, fault);
+	if (fault->prm.flags & IOMMU_FAULT_PAGE_REQUEST_PASID_VALID) {
-	if (!group->domain)
+		group->attach_handle = iommu_attach_handle_get(dev->iommu_group,
+							       fault->prm.pasid,
+							       0);
+		if (IS_ERR(group->attach_handle)) {
+			const struct iommu_ops *ops = dev_iommu_ops(dev);
+			if (!ops->user_pasid_table)
+				goto err_abort;
+			/*
+			 * The iommu driver for this device supports user-
+			 * managed PASID table. Therefore page faults for
+			 * any PASID should go through the NESTING domain
+			 * attached to the device RID.
+			 */
+			group->attach_handle =
+				iommu_attach_handle_get(dev->iommu_group,
+							IOMMU_NO_PASID,
+							IOMMU_DOMAIN_NESTED);
+			if (IS_ERR(group->attach_handle))
+				goto err_abort;
+		}
+	} else {
+		group->attach_handle =
+			iommu_attach_handle_get(dev->iommu_group, IOMMU_NO_PASID, 0);
+		if (IS_ERR(group->attach_handle))
+			goto err_abort;
+	}
+	if (!group->attach_handle->domain->iopf_handler)
 		goto err_abort;
 	/*
 	 * On success iopf_handler must call iopf_group_response() and
 	 * iopf_free_group()
 	 */
-	if (group->domain->iopf_handler(group))
+	if (group->attach_handle->domain->iopf_handler(group))
 		goto err_abort;
 	return;
 err_abort:
+	dev_warn_ratelimited(dev, "iopf with pasid %d aborted\n",
+			     fault->prm.pasid);
 	iopf_group_response(group, IOMMU_PAGE_RESP_FAILURE);
 	if (group == &abort_group)
 		__iopf_free_group(group);

drivers/iommu/iommu-priv.h+11−0

@@ -28,4 +28,15 @@ void iommu_device_unregister_bus(struct iommu_device *iommu, · 27 unmodified lines
28	const struct bus_type *bus,	28	const struct bus_type *bus,
29	struct notifier_block *nb);	29	struct notifier_block *nb);
30		30
		31	struct iommu_attach_handle iommu_attach_handle_get(struct iommu_group group,
		32	ioasid_t pasid,
		33	unsigned int type);
		34	int iommu_attach_group_handle(struct iommu_domain *domain,
		35	struct iommu_group *group,
		36	struct iommu_attach_handle *handle);
		37	void iommu_detach_group_handle(struct iommu_domain *domain,
		38	struct iommu_group *group);
		39	int iommu_replace_group_handle(struct iommu_group *group,
		40	struct iommu_domain *new_domain,
		41	struct iommu_attach_handle *handle);
31	#endif /* __LINUX_IOMMU_PRIV_H */	42	#endif /* __LINUX_IOMMU_PRIV_H */

drivers/iommu/iommu-sva.c+28−14

 	}
 	iommu_mm->pasid = pasid;
 	INIT_LIST_HEAD(&iommu_mm->sva_domains);
-	INIT_LIST_HEAD(&iommu_mm->sva_handles);
 	/*
 	 * Make sure the write to mm->iommu_mm is not reordered in front of
 	 * initialization to iommu_mm fields. If it does, readers may see a
  */
 struct iommu_sva *iommu_sva_bind_device(struct device *dev, struct mm_struct *mm)
 {
+	struct iommu_group *group = dev->iommu_group;
+	struct iommu_attach_handle *attach_handle;
 	struct iommu_mm_data *iommu_mm;
 	struct iommu_domain *domain;
 	struct iommu_sva *handle;
 	int ret;
+	if (!group)
+		return ERR_PTR(-ENODEV);
 	mutex_lock(&iommu_sva_lock);
 	/* Allocate mm->pasid if necessary. */
 		goto out_unlock;
 	}
-	list_for_each_entry(handle, &mm->iommu_mm->sva_handles, handle_item) {
+	/* A bond already exists, just take a reference`. */
-		if (handle->dev == dev) {
+	attach_handle = iommu_attach_handle_get(group, iommu_mm->pasid, IOMMU_DOMAIN_SVA);
-			refcount_inc(&handle->users);
+	if (!IS_ERR(attach_handle)) {
-			mutex_unlock(&iommu_sva_lock);
+		handle = container_of(attach_handle, struct iommu_sva, handle);
-			return handle;
+		if (attach_handle->domain->mm != mm) {
+			ret = -EBUSY;
+			goto out_unlock;
 		}
+		refcount_inc(&handle->users);
+		mutex_unlock(&iommu_sva_lock);
+		return handle;
+	}
+	if (PTR_ERR(attach_handle) != -ENOENT) {
+		ret = PTR_ERR(attach_handle);
+		goto out_unlock;
 	}
 	handle = kzalloc(sizeof(*handle), GFP_KERNEL);
 	/* Search for an existing domain. */
 	list_for_each_entry(domain, &mm->iommu_mm->sva_domains, next) {
-		ret = iommu_attach_device_pasid(domain, dev, iommu_mm->pasid);
+		ret = iommu_attach_device_pasid(domain, dev, iommu_mm->pasid,
+						&handle->handle);
 		if (!ret) {
 			domain->users++;
 			goto out;
 		goto out_free_handle;
 	}
-	ret = iommu_attach_device_pasid(domain, dev, iommu_mm->pasid);
+	ret = iommu_attach_device_pasid(domain, dev, iommu_mm->pasid,
+					&handle->handle);
 	if (ret)
 		goto out_free_domain;
 	domain->users = 1;
 out:
 	refcount_set(&handle->users, 1);
-	list_add(&handle->handle_item, &mm->iommu_mm->sva_handles);
 	mutex_unlock(&iommu_sva_lock);
 	handle->dev = dev;
-	handle->domain = domain;
 	return handle;
 out_free_domain:
  */
 void iommu_sva_unbind_device(struct iommu_sva *handle)
 {
-	struct iommu_domain *domain = handle->domain;
+	struct iommu_domain *domain = handle->handle.domain;
 	struct iommu_mm_data *iommu_mm = domain->mm->iommu_mm;
 	struct device *dev = handle->dev;
 		mutex_unlock(&iommu_sva_lock);
 		return;
 	}
-	list_del(&handle->handle_item);
 	iommu_detach_device_pasid(domain, dev, iommu_mm->pasid);
 	if (--domain->users == 0) {
 u32 iommu_sva_get_pasid(struct iommu_sva *handle)
 {
-	struct iommu_domain *domain = handle->domain;
+	struct iommu_domain *domain = handle->handle.domain;
 	return mm_get_enqcmd_pasid(domain->mm);
 }
 		if (status != IOMMU_PAGE_RESP_SUCCESS)
 			break;
-		status = iommu_sva_handle_mm(&iopf->fault, group->domain->mm);
+		status = iommu_sva_handle_mm(&iopf->fault,
+					     group->attach_handle->domain->mm);
 	}
 	iopf_group_response(group, status);

drivers/iommu/iommu.c+143−42

@@ -3352,16 +3352,17 @@ static void __iommu_remove_group_pasid(struct iommu_group *group, · 3351 unmodified lines
3352	* @domain: the iommu domain.	3352	* @domain: the iommu domain.
3353	* @dev: the attached device.	3353	* @dev: the attached device.
3354	* @pasid: the pasid of the device.	3354	* @pasid: the pasid of the device.
		3355	* @handle: the attach handle.
3355	*	3356	*
3356	* Return: 0 on success, or an error.	3357	* Return: 0 on success, or an error.
3357	*/	3358	*/
3358	int iommu_attach_device_pasid(struct iommu_domain *domain,	3359	int iommu_attach_device_pasid(struct iommu_domain *domain,
3359	struct device *dev, ioasid_t pasid)	3360	struct device *dev, ioasid_t pasid,
		3361	struct iommu_attach_handle *handle)
3360	{	3362	{
3361	/* Caller must be a probed driver on dev */	3363	/* Caller must be a probed driver on dev */
3362	struct iommu_group *group = dev->iommu_group;	3364	struct iommu_group *group = dev->iommu_group;
3363	struct group_device *device;	3365	struct group_device *device;
3364	void *curr;
3365	int ret;	3366	int ret;
3366		3367
3367	if (!domain->ops->set_dev_pasid)	3368	if (!domain->ops->set_dev_pasid)
@@ -3382,11 +3383,12 @@ int iommu_attach_device_pasid(struct iommu_domain *domain, · 14 unmodified lines
3382	}	3383	}
3383	}	3384	}
3384		3385
3385	curr = xa_cmpxchg(&group->pasid_array, pasid, NULL, domain, GFP_KERNEL);	3386	if (handle)
3386	if (curr) {	3387	handle->domain = domain;
3387	ret = xa_err(curr) ? : -EBUSY;	3388
		3389	ret = xa_insert(&group->pasid_array, pasid, handle, GFP_KERNEL);
		3390	if (ret)
3388	goto out_unlock;	3391	goto out_unlock;
3389	}
3390		3392
3391	ret = __iommu_set_group_pasid(domain, group, pasid);	3393	ret = __iommu_set_group_pasid(domain, group, pasid);
3392	if (ret)	3394	if (ret)
@@ -3414,46 +3416,11 @@ void iommu_detach_device_pasid(struct iommu_domain domain, struct device dev, · 21 unmodified lines
3414		3416
3415	mutex_lock(&group->mutex);	3417	mutex_lock(&group->mutex);
3416	__iommu_remove_group_pasid(group, pasid, domain);	3418	__iommu_remove_group_pasid(group, pasid, domain);
3417	WARN_ON(xa_erase(&group->pasid_array, pasid) != domain);	3419	xa_erase(&group->pasid_array, pasid);
3418	mutex_unlock(&group->mutex);	3420	mutex_unlock(&group->mutex);
3419	}	3421	}
3420	EXPORT_SYMBOL_GPL(iommu_detach_device_pasid);	3422	EXPORT_SYMBOL_GPL(iommu_detach_device_pasid);
3421		3423
3422	/*
3423	* iommu_get_domain_for_dev_pasid() - Retrieve domain for @pasid of @dev
3424	* @dev: the queried device
3425	* @pasid: the pasid of the device
3426	* @type: matched domain type, 0 for any match
3427	*
3428	* This is a variant of iommu_get_domain_for_dev(). It returns the existing
3429	* domain attached to pasid of a device. Callers must hold a lock around this
3430	* function, and both iommu_attach/detach_dev_pasid() whenever a domain of
3431	* type is being manipulated. This API does not internally resolve races with
3432	* attach/detach.
3433	*
3434	* Return: attached domain on success, NULL otherwise.
3435	*/
3436	struct iommu_domain iommu_get_domain_for_dev_pasid(struct device dev,
3437	ioasid_t pasid,
3438	unsigned int type)
3439	{
3440	/* Caller must be a probed driver on dev */
3441	struct iommu_group *group = dev->iommu_group;
3442	struct iommu_domain *domain;
3443
3444	if (!group)
3445	return NULL;
3446
3447	xa_lock(&group->pasid_array);
3448	domain = xa_load(&group->pasid_array, pasid);
3449	if (type && domain && domain->type != type)
3450	domain = ERR_PTR(-EBUSY);
3451	xa_unlock(&group->pasid_array);
3452
3453	return domain;
3454	}
3455	EXPORT_SYMBOL_GPL(iommu_get_domain_for_dev_pasid);
3456
3457	ioasid_t iommu_alloc_global_pasid(struct device *dev)	3424	ioasid_t iommu_alloc_global_pasid(struct device *dev)
3458	{	3425	{
3459	int ret;	3426	int ret;
@@ -3480,3 +3447,137 @@ void iommu_free_global_pasid(ioasid_t pasid) · 20 unmodified lines
3480	ida_free(&iommu_global_pasid_ida, pasid);	3447	ida_free(&iommu_global_pasid_ida, pasid);
3481	}	3448	}
3482	EXPORT_SYMBOL_GPL(iommu_free_global_pasid);	3449	EXPORT_SYMBOL_GPL(iommu_free_global_pasid);
		3450
		3451	/**
		3452	* iommu_attach_handle_get - Return the attach handle
		3453	* @group: the iommu group that domain was attached to
		3454	* @pasid: the pasid within the group
		3455	* @type: matched domain type, 0 for any match
		3456	*
		3457	* Return handle or ERR_PTR(-ENOENT) on none, ERR_PTR(-EBUSY) on mismatch.
		3458	*
		3459	* Return the attach handle to the caller. The life cycle of an iommu attach
		3460	* handle is from the time when the domain is attached to the time when the
		3461	* domain is detached. Callers are required to synchronize the call of
		3462	* iommu_attach_handle_get() with domain attachment and detachment. The attach
		3463	* handle can only be used during its life cycle.
		3464	*/
		3465	struct iommu_attach_handle *
		3466	iommu_attach_handle_get(struct iommu_group *group, ioasid_t pasid, unsigned int type)
		3467	{
		3468	struct iommu_attach_handle *handle;
		3469
		3470	xa_lock(&group->pasid_array);
		3471	handle = xa_load(&group->pasid_array, pasid);
		3472	if (!handle)
		3473	handle = ERR_PTR(-ENOENT);
		3474	else if (type && handle->domain->type != type)
		3475	handle = ERR_PTR(-EBUSY);
		3476	xa_unlock(&group->pasid_array);
		3477
		3478	return handle;
		3479	}
		3480	EXPORT_SYMBOL_NS_GPL(iommu_attach_handle_get, IOMMUFD_INTERNAL);
		3481
		3482	/**
		3483	* iommu_attach_group_handle - Attach an IOMMU domain to an IOMMU group
		3484	* @domain: IOMMU domain to attach
		3485	* @group: IOMMU group that will be attached
		3486	* @handle: attach handle
		3487	*
		3488	* Returns 0 on success and error code on failure.
		3489	*
		3490	* This is a variant of iommu_attach_group(). It allows the caller to provide
		3491	* an attach handle and use it when the domain is attached. This is currently
		3492	* used by IOMMUFD to deliver the I/O page faults.
		3493	*/
		3494	int iommu_attach_group_handle(struct iommu_domain *domain,
		3495	struct iommu_group *group,
		3496	struct iommu_attach_handle *handle)
		3497	{
		3498	int ret;
		3499
		3500	if (handle)
		3501	handle->domain = domain;
		3502
		3503	mutex_lock(&group->mutex);
		3504	ret = xa_insert(&group->pasid_array, IOMMU_NO_PASID, handle, GFP_KERNEL);
		3505	if (ret)
		3506	goto err_unlock;
		3507
		3508	ret = __iommu_attach_group(domain, group);
		3509	if (ret)
		3510	goto err_erase;
		3511	mutex_unlock(&group->mutex);
		3512
		3513	return 0;
		3514	err_erase:
		3515	xa_erase(&group->pasid_array, IOMMU_NO_PASID);
		3516	err_unlock:
		3517	mutex_unlock(&group->mutex);
		3518	return ret;
		3519	}
		3520	EXPORT_SYMBOL_NS_GPL(iommu_attach_group_handle, IOMMUFD_INTERNAL);
		3521
		3522	/**
		3523	* iommu_detach_group_handle - Detach an IOMMU domain from an IOMMU group
		3524	* @domain: IOMMU domain to attach
		3525	* @group: IOMMU group that will be attached
		3526	*
		3527	* Detach the specified IOMMU domain from the specified IOMMU group.
		3528	* It must be used in conjunction with iommu_attach_group_handle().
		3529	*/
		3530	void iommu_detach_group_handle(struct iommu_domain *domain,
		3531	struct iommu_group *group)
		3532	{
		3533	mutex_lock(&group->mutex);
		3534	__iommu_group_set_core_domain(group);
		3535	xa_erase(&group->pasid_array, IOMMU_NO_PASID);
		3536	mutex_unlock(&group->mutex);
		3537	}
		3538	EXPORT_SYMBOL_NS_GPL(iommu_detach_group_handle, IOMMUFD_INTERNAL);
		3539
		3540	/**
		3541	* iommu_replace_group_handle - replace the domain that a group is attached to
		3542	* @group: IOMMU group that will be attached to the new domain
		3543	* @new_domain: new IOMMU domain to replace with
		3544	* @handle: attach handle
		3545	*
		3546	* This is a variant of iommu_group_replace_domain(). It allows the caller to
		3547	* provide an attach handle for the new domain and use it when the domain is
		3548	* attached.
		3549	*/
		3550	int iommu_replace_group_handle(struct iommu_group *group,
		3551	struct iommu_domain *new_domain,
		3552	struct iommu_attach_handle *handle)
		3553	{
		3554	void *curr;
		3555	int ret;
		3556
		3557	if (!new_domain)
		3558	return -EINVAL;
		3559
		3560	mutex_lock(&group->mutex);
		3561	if (handle) {
		3562	ret = xa_reserve(&group->pasid_array, IOMMU_NO_PASID, GFP_KERNEL);
		3563	if (ret)
		3564	goto err_unlock;
		3565	}
		3566
		3567	ret = __iommu_group_set_domain(group, new_domain);
		3568	if (ret)
		3569	goto err_release;
		3570
		3571	curr = xa_store(&group->pasid_array, IOMMU_NO_PASID, handle, GFP_KERNEL);
		3572	WARN_ON(xa_is_err(curr));
		3573
		3574	mutex_unlock(&group->mutex);
		3575
		3576	return 0;
		3577	err_release:
		3578	xa_release(&group->pasid_array, IOMMU_NO_PASID);
		3579	err_unlock:
		3580	mutex_unlock(&group->mutex);
		3581	return ret;
		3582	}
		3583	EXPORT_SYMBOL_NS_GPL(iommu_replace_group_handle, IOMMUFD_INTERNAL);

drivers/iommu/iommufd/Makefile+1−0

@@ -1,6 +1,7 @@
1	# SPDX-License-Identifier: GPL-2.0-only	1	# SPDX-License-Identifier: GPL-2.0-only
2	iommufd-y := \	2	iommufd-y := \
3	device.o \	3	device.o \
		4	fault.o \
4	hw_pagetable.o \	5	hw_pagetable.o \
5	io_pagetable.o \	6	io_pagetable.o \
6	ioas.o \	7	ioas.o \

drivers/iommu/iommufd/device.c+4−3

 	refcount_inc(&idev->obj.users);
 	/* igroup refcount moves into iommufd_device */
 	idev->igroup = igroup;
+	mutex_init(&idev->iopf_lock);
 	/*
 	 * If the caller fails after this success it must call
 	 * attachment.
 	 */
 	if (list_empty(&idev->igroup->device_list)) {
-		rc = iommu_attach_group(hwpt->domain, idev->igroup->group);
+		rc = iommufd_hwpt_attach_device(hwpt, idev);
 		if (rc)
 			goto err_unresv;
 		idev->igroup->hwpt = hwpt;
 	mutex_lock(&idev->igroup->lock);
 	list_del(&idev->group_item);
 	if (list_empty(&idev->igroup->device_list)) {
-		iommu_detach_group(hwpt->domain, idev->igroup->group);
+		iommufd_hwpt_detach_device(hwpt, idev);
 		idev->igroup->hwpt = NULL;
 	}
 	if (hwpt_is_paging(hwpt))
 			goto err_unlock;
 	}
-	rc = iommu_group_replace_domain(igroup->group, hwpt->domain);
+	rc = iommufd_hwpt_replace_device(idev, hwpt, old_hwpt);
 	if (rc)
 		goto err_unresv;

drivers/iommu/iommufd/fault.c+443−0

@@ -0,0 +1,443 @@
	1	// SPDX-License-Identifier: GPL-2.0-only
	2	/* Copyright (C) 2024 Intel Corporation
	3	*/
	4	#define pr_fmt(fmt) "iommufd: " fmt
	5
	6	#include <linux/file.h>
	7	#include <linux/fs.h>
	8	#include <linux/module.h>
	9	#include <linux/mutex.h>
	10	#include <linux/iommufd.h>
	11	#include <linux/pci.h>
	12	#include <linux/poll.h>
	13	#include <linux/anon_inodes.h>
	14	#include <uapi/linux/iommufd.h>
	15
	16	#include "../iommu-priv.h"
	17	#include "iommufd_private.h"
	18
	19	static int iommufd_fault_iopf_enable(struct iommufd_device *idev)
	20	{
	21	struct device *dev = idev->dev;
	22	int ret;
	23
	24	/*
	25	* Once we turn on PCI/PRI support for VF, the response failure code
	26	* should not be forwarded to the hardware due to PRI being a shared
	27	* resource between PF and VFs. There is no coordination for this
	28	* shared capability. This waits for a vPRI reset to recover.
	29	*/
	30	if (dev_is_pci(dev) && to_pci_dev(dev)->is_virtfn)
	31	return -EINVAL;
	32
	33	mutex_lock(&idev->iopf_lock);
	34	/* Device iopf has already been on. */
	35	if (++idev->iopf_enabled > 1) {
	36	mutex_unlock(&idev->iopf_lock);
	37	return 0;
	38	}
	39
	40	ret = iommu_dev_enable_feature(dev, IOMMU_DEV_FEAT_IOPF);
	41	if (ret)
	42	--idev->iopf_enabled;
	43	mutex_unlock(&idev->iopf_lock);
	44
	45	return ret;
	46	}
	47
	48	static void iommufd_fault_iopf_disable(struct iommufd_device *idev)
	49	{
	50	mutex_lock(&idev->iopf_lock);
	51	if (!WARN_ON(idev->iopf_enabled == 0)) {
	52	if (--idev->iopf_enabled == 0)
	53	iommu_dev_disable_feature(idev->dev, IOMMU_DEV_FEAT_IOPF);
	54	}
	55	mutex_unlock(&idev->iopf_lock);
	56	}
	57
	58	static int __fault_domain_attach_dev(struct iommufd_hw_pagetable *hwpt,
	59	struct iommufd_device *idev)
	60	{
	61	struct iommufd_attach_handle *handle;
	62	int ret;
	63
	64	handle = kzalloc(sizeof(*handle), GFP_KERNEL);
	65	if (!handle)
	66	return -ENOMEM;
	67
	68	handle->idev = idev;
	69	ret = iommu_attach_group_handle(hwpt->domain, idev->igroup->group,
	70	&handle->handle);
	71	if (ret)
	72	kfree(handle);
	73
	74	return ret;
	75	}
	76
	77	int iommufd_fault_domain_attach_dev(struct iommufd_hw_pagetable *hwpt,
	78	struct iommufd_device *idev)
	79	{
	80	int ret;
	81
	82	if (!hwpt->fault)
	83	return -EINVAL;
	84
	85	ret = iommufd_fault_iopf_enable(idev);
	86	if (ret)
	87	return ret;
	88
	89	ret = __fault_domain_attach_dev(hwpt, idev);
	90	if (ret)
	91	iommufd_fault_iopf_disable(idev);
	92
	93	return ret;
	94	}
	95
	96	static void iommufd_auto_response_faults(struct iommufd_hw_pagetable *hwpt,
	97	struct iommufd_attach_handle *handle)
	98	{
	99	struct iommufd_fault *fault = hwpt->fault;
	100	struct iopf_group group, next;
	101	unsigned long index;
	102
	103	if (!fault)
	104	return;
	105
	106	mutex_lock(&fault->mutex);
	107	list_for_each_entry_safe(group, next, &fault->deliver, node) {
	108	if (group->attach_handle != &handle->handle)
	109	continue;
	110	list_del(&group->node);
	111	iopf_group_response(group, IOMMU_PAGE_RESP_INVALID);
	112	iopf_free_group(group);
	113	}
	114
	115	xa_for_each(&fault->response, index, group) {
	116	if (group->attach_handle != &handle->handle)
	117	continue;
	118	xa_erase(&fault->response, index);
	119	iopf_group_response(group, IOMMU_PAGE_RESP_INVALID);
	120	iopf_free_group(group);
	121	}
	122	mutex_unlock(&fault->mutex);
	123	}
	124
	125	static struct iommufd_attach_handle *
	126	iommufd_device_get_attach_handle(struct iommufd_device *idev)
	127	{
	128	struct iommu_attach_handle *handle;
	129
	130	handle = iommu_attach_handle_get(idev->igroup->group, IOMMU_NO_PASID, 0);
	131	if (IS_ERR(handle))
	132	return NULL;
	133
	134	return to_iommufd_handle(handle);
	135	}
	136
	137	void iommufd_fault_domain_detach_dev(struct iommufd_hw_pagetable *hwpt,
	138	struct iommufd_device *idev)
	139	{
	140	struct iommufd_attach_handle *handle;
	141
	142	handle = iommufd_device_get_attach_handle(idev);
	143	iommu_detach_group_handle(hwpt->domain, idev->igroup->group);
	144	iommufd_auto_response_faults(hwpt, handle);
	145	iommufd_fault_iopf_disable(idev);
	146	kfree(handle);
	147	}
	148
	149	static int __fault_domain_replace_dev(struct iommufd_device *idev,
	150	struct iommufd_hw_pagetable *hwpt,
	151	struct iommufd_hw_pagetable *old)
	152	{
	153	struct iommufd_attach_handle handle, curr = NULL;
	154	int ret;
	155
	156	if (old->fault)
	157	curr = iommufd_device_get_attach_handle(idev);
	158
	159	if (hwpt->fault) {
	160	handle = kzalloc(sizeof(*handle), GFP_KERNEL);
	161	if (!handle)
	162	return -ENOMEM;
	163
	164	handle->handle.domain = hwpt->domain;
	165	handle->idev = idev;
	166	ret = iommu_replace_group_handle(idev->igroup->group,
	167	hwpt->domain, &handle->handle);
	168	} else {
	169	ret = iommu_replace_group_handle(idev->igroup->group,
	170	hwpt->domain, NULL);
	171	}
	172
	173	if (!ret && curr) {
	174	iommufd_auto_response_faults(old, curr);
	175	kfree(curr);
	176	}
	177
	178	return ret;
	179	}
	180
	181	int iommufd_fault_domain_replace_dev(struct iommufd_device *idev,
	182	struct iommufd_hw_pagetable *hwpt,
	183	struct iommufd_hw_pagetable *old)
	184	{
	185	bool iopf_off = !hwpt->fault && old->fault;
	186	bool iopf_on = hwpt->fault && !old->fault;
	187	int ret;
	188
	189	if (iopf_on) {
	190	ret = iommufd_fault_iopf_enable(idev);
	191	if (ret)
	192	return ret;
	193	}
	194
	195	ret = __fault_domain_replace_dev(idev, hwpt, old);
	196	if (ret) {
	197	if (iopf_on)
	198	iommufd_fault_iopf_disable(idev);
	199	return ret;
	200	}
	201
	202	if (iopf_off)
	203	iommufd_fault_iopf_disable(idev);
	204
	205	return 0;
	206	}
	207
	208	void iommufd_fault_destroy(struct iommufd_object *obj)
	209	{
	210	struct iommufd_fault *fault = container_of(obj, struct iommufd_fault, obj);
	211	struct iopf_group group, next;
	212
	213	/*
	214	* The iommufd object's reference count is zero at this point.
	215	* We can be confident that no other threads are currently
	216	* accessing this pointer. Therefore, acquiring the mutex here
	217	* is unnecessary.
	218	*/
	219	list_for_each_entry_safe(group, next, &fault->deliver, node) {
	220	list_del(&group->node);
	221	iopf_group_response(group, IOMMU_PAGE_RESP_INVALID);
	222	iopf_free_group(group);
	223	}
	224	}
	225
	226	static void iommufd_compose_fault_message(struct iommu_fault *fault,
	227	struct iommu_hwpt_pgfault *hwpt_fault,
	228	struct iommufd_device *idev,
	229	u32 cookie)
	230	{
	231	hwpt_fault->flags = fault->prm.flags;
	232	hwpt_fault->dev_id = idev->obj.id;
	233	hwpt_fault->pasid = fault->prm.pasid;
	234	hwpt_fault->grpid = fault->prm.grpid;
	235	hwpt_fault->perm = fault->prm.perm;
	236	hwpt_fault->addr = fault->prm.addr;
	237	hwpt_fault->length = 0;
	238	hwpt_fault->cookie = cookie;
	239	}
	240
	241	static ssize_t iommufd_fault_fops_read(struct file filep, char __user buf,
	242	size_t count, loff_t *ppos)
	243	{
	244	size_t fault_size = sizeof(struct iommu_hwpt_pgfault);
	245	struct iommufd_fault *fault = filep->private_data;
	246	struct iommu_hwpt_pgfault data;
	247	struct iommufd_device *idev;
	248	struct iopf_group *group;
	249	struct iopf_fault *iopf;
	250	size_t done = 0;
	251	int rc = 0;
	252
	253	if (*ppos \|\| count % fault_size)
	254	return -ESPIPE;
	255
	256	mutex_lock(&fault->mutex);
	257	while (!list_empty(&fault->deliver) && count > done) {
	258	group = list_first_entry(&fault->deliver,
	259	struct iopf_group, node);
	260
	261	if (group->fault_count * fault_size > count - done)
	262	break;
	263
	264	rc = xa_alloc(&fault->response, &group->cookie, group,
	265	xa_limit_32b, GFP_KERNEL);
	266	if (rc)
	267	break;
	268
	269	idev = to_iommufd_handle(group->attach_handle)->idev;
	270	list_for_each_entry(iopf, &group->faults, list) {
	271	iommufd_compose_fault_message(&iopf->fault,
	272	&data, idev,
	273	group->cookie);
	274	if (copy_to_user(buf + done, &data, fault_size)) {
	275	xa_erase(&fault->response, group->cookie);
	276	rc = -EFAULT;
	277	break;
	278	}
	279	done += fault_size;
	280	}
	281
	282	list_del(&group->node);
	283	}
	284	mutex_unlock(&fault->mutex);
	285
	286	return done == 0 ? rc : done;
	287	}
	288
	289	static ssize_t iommufd_fault_fops_write(struct file filep, const char __user buf,
	290	size_t count, loff_t *ppos)
	291	{
	292	size_t response_size = sizeof(struct iommu_hwpt_page_response);
	293	struct iommufd_fault *fault = filep->private_data;
	294	struct iommu_hwpt_page_response response;
	295	struct iopf_group *group;
	296	size_t done = 0;
	297	int rc = 0;
	298
	299	if (*ppos \|\| count % response_size)
	300	return -ESPIPE;
	301
	302	mutex_lock(&fault->mutex);
	303	while (count > done) {
	304	rc = copy_from_user(&response, buf + done, response_size);
	305	if (rc)
	306	break;
	307
	308	static_assert((int)IOMMUFD_PAGE_RESP_SUCCESS ==
	309	(int)IOMMU_PAGE_RESP_SUCCESS);
	310	static_assert((int)IOMMUFD_PAGE_RESP_INVALID ==
	311	(int)IOMMU_PAGE_RESP_INVALID);
	312	if (response.code != IOMMUFD_PAGE_RESP_SUCCESS &&
	313	response.code != IOMMUFD_PAGE_RESP_INVALID) {
	314	rc = -EINVAL;
	315	break;
	316	}
	317
	318	group = xa_erase(&fault->response, response.cookie);
	319	if (!group) {
	320	rc = -EINVAL;
	321	break;
	322	}
	323
	324	iopf_group_response(group, response.code);
	325	iopf_free_group(group);
	326	done += response_size;
	327	}
	328	mutex_unlock(&fault->mutex);
	329
	330	return done == 0 ? rc : done;
	331	}
	332
	333	static __poll_t iommufd_fault_fops_poll(struct file *filep,
	334	struct poll_table_struct *wait)
	335	{
	336	struct iommufd_fault *fault = filep->private_data;
	337	__poll_t pollflags = EPOLLOUT;
	338
	339	poll_wait(filep, &fault->wait_queue, wait);
	340	mutex_lock(&fault->mutex);
	341	if (!list_empty(&fault->deliver))
	342	pollflags \|= EPOLLIN \| EPOLLRDNORM;
	343	mutex_unlock(&fault->mutex);
	344
	345	return pollflags;
	346	}
	347
	348	static int iommufd_fault_fops_release(struct inode inode, struct file filep)
	349	{
	350	struct iommufd_fault *fault = filep->private_data;
	351
	352	refcount_dec(&fault->obj.users);
	353	iommufd_ctx_put(fault->ictx);
	354	return 0;
	355	}
	356
	357	static const struct file_operations iommufd_fault_fops = {
	358	.owner = THIS_MODULE,
	359	.open = nonseekable_open,
	360	.read = iommufd_fault_fops_read,
	361	.write = iommufd_fault_fops_write,
	362	.poll = iommufd_fault_fops_poll,
	363	.release = iommufd_fault_fops_release,
	364	.llseek = no_llseek,
	365	};
	366
	367	int iommufd_fault_alloc(struct iommufd_ucmd *ucmd)
	368	{
	369	struct iommu_fault_alloc *cmd = ucmd->cmd;
	370	struct iommufd_fault *fault;
	371	struct file *filep;
	372	int fdno;
	373	int rc;
	374
	375	if (cmd->flags)
	376	return -EOPNOTSUPP;
	377
	378	fault = iommufd_object_alloc(ucmd->ictx, fault, IOMMUFD_OBJ_FAULT);
	379	if (IS_ERR(fault))
	380	return PTR_ERR(fault);
	381
	382	fault->ictx = ucmd->ictx;
	383	INIT_LIST_HEAD(&fault->deliver);
	384	xa_init_flags(&fault->response, XA_FLAGS_ALLOC1);
	385	mutex_init(&fault->mutex);
	386	init_waitqueue_head(&fault->wait_queue);
	387
	388	filep = anon_inode_getfile("[iommufd-pgfault]", &iommufd_fault_fops,
	389	fault, O_RDWR);
	390	if (IS_ERR(filep)) {
	391	rc = PTR_ERR(filep);
	392	goto out_abort;
	393	}
	394
	395	refcount_inc(&fault->obj.users);
	396	iommufd_ctx_get(fault->ictx);
	397	fault->filep = filep;
	398
	399	fdno = get_unused_fd_flags(O_CLOEXEC);
	400	if (fdno < 0) {
	401	rc = fdno;
	402	goto out_fput;
	403	}
	404
	405	cmd->out_fault_id = fault->obj.id;
	406	cmd->out_fault_fd = fdno;
	407
	408	rc = iommufd_ucmd_respond(ucmd, sizeof(*cmd));
	409	if (rc)
	410	goto out_put_fdno;
	411	iommufd_object_finalize(ucmd->ictx, &fault->obj);
	412
	413	fd_install(fdno, fault->filep);
	414
	415	return 0;
	416	out_put_fdno:
	417	put_unused_fd(fdno);
	418	out_fput:
	419	fput(filep);
	420	refcount_dec(&fault->obj.users);
	421	iommufd_ctx_put(fault->ictx);
	422	out_abort:
	423	iommufd_object_abort_and_destroy(ucmd->ictx, &fault->obj);
	424
	425	return rc;
	426	}
	427
	428	int iommufd_fault_iopf_handler(struct iopf_group *group)
	429	{
	430	struct iommufd_hw_pagetable *hwpt;
	431	struct iommufd_fault *fault;
	432
	433	hwpt = group->attach_handle->domain->fault_data;
	434	fault = hwpt->fault;
	435
	436	mutex_lock(&fault->mutex);
	437	list_add_tail(&group->node, &fault->deliver);
	438	mutex_unlock(&fault->mutex);
	439
	440	wake_up_interruptible(&fault->wait_queue);
	441
	442	return 0;
	443	}

drivers/iommu/iommufd/hw_pagetable.c+32−9

 #include "../iommu-priv.h"
 #include "iommufd_private.h"
+static void __iommufd_hwpt_destroy(struct iommufd_hw_pagetable *hwpt)
+{
+	if (hwpt->domain)
+		iommu_domain_free(hwpt->domain);
+	if (hwpt->fault)
+		refcount_dec(&hwpt->fault->obj.users);
+}
 void iommufd_hwpt_paging_destroy(struct iommufd_object *obj)
 {
 	struct iommufd_hwpt_paging *hwpt_paging =
 					 hwpt_paging->common.domain);
 	}
-	if (hwpt_paging->common.domain)
+	__iommufd_hwpt_destroy(&hwpt_paging->common);
-		iommu_domain_free(hwpt_paging->common.domain);
 	refcount_dec(&hwpt_paging->ioas->obj.users);
 }
 	struct iommufd_hwpt_nested *hwpt_nested =
 		container_of(obj, struct iommufd_hwpt_nested, common.obj);
-	if (hwpt_nested->common.domain)
+	__iommufd_hwpt_destroy(&hwpt_nested->common);
-		iommu_domain_free(hwpt_nested->common.domain);
 	refcount_dec(&hwpt_nested->parent->common.obj.users);
 }
 	struct iommufd_hw_pagetable *hwpt;
 	int rc;
-	if (flags || !user_data->len || !ops->domain_alloc_user)
+	if ((flags & ~IOMMU_HWPT_FAULT_ID_VALID) ||
+	    !user_data->len || !ops->domain_alloc_user)
 		return ERR_PTR(-EOPNOTSUPP);
 	if (parent->auto_domain || !parent->nest_parent)
 		return ERR_PTR(-EINVAL);
 	refcount_inc(&parent->common.obj.users);
 	hwpt_nested->parent = parent;
-	hwpt->domain = ops->domain_alloc_user(idev->dev, flags,
+	hwpt->domain = ops->domain_alloc_user(idev->dev,
+					      flags & ~IOMMU_HWPT_FAULT_ID_VALID,
 					      parent->common.domain, user_data);
 	if (IS_ERR(hwpt->domain)) {
 		rc = PTR_ERR(hwpt->domain);
 	}
 	hwpt->domain->owner = ops;
-	if (WARN_ON_ONCE(hwpt->domain->type != IOMMU_DOMAIN_NESTED)) {
+	if (WARN_ON_ONCE(hwpt->domain->type != IOMMU_DOMAIN_NESTED ||
+			 !hwpt->domain->ops->cache_invalidate_user)) {
 		rc = -EINVAL;
 		goto out_abort;
 	}
 		goto out_put_pt;
 	}
+	if (cmd->flags & IOMMU_HWPT_FAULT_ID_VALID) {
+		struct iommufd_fault *fault;
+		fault = iommufd_get_fault(ucmd, cmd->fault_id);
+		if (IS_ERR(fault)) {
+			rc = PTR_ERR(fault);
+			goto out_hwpt;
+		}
+		hwpt->fault = fault;
+		hwpt->domain->iopf_handler = iommufd_fault_iopf_handler;
+		hwpt->domain->fault_data = hwpt;
+		refcount_inc(&fault->obj.users);
+		iommufd_put_object(ucmd->ictx, &fault->obj);
+	}
 	cmd->out_hwpt_id = hwpt->obj.id;
 	rc = iommufd_ucmd_respond(ucmd, sizeof(*cmd));
 	if (rc)

drivers/iommu/iommufd/iommufd_private.h+80−0

@@ -11,6 +11,7 @@ · 10 unmodified lines
11	#include <linux/iommu.h>	11	#include <linux/iommu.h>
12	#include <linux/iova_bitmap.h>	12	#include <linux/iova_bitmap.h>
13	#include <uapi/linux/iommufd.h>	13	#include <uapi/linux/iommufd.h>
		14	#include "../iommu-priv.h"
14		15
15	struct iommu_domain;	16	struct iommu_domain;
16	struct iommu_group;	17	struct iommu_group;
@@ -128,6 +129,7 @@ enum iommufd_object_type { · 111 unmodified lines
128	IOMMUFD_OBJ_HWPT_NESTED,	129	IOMMUFD_OBJ_HWPT_NESTED,
129	IOMMUFD_OBJ_IOAS,	130	IOMMUFD_OBJ_IOAS,
130	IOMMUFD_OBJ_ACCESS,	131	IOMMUFD_OBJ_ACCESS,
		132	IOMMUFD_OBJ_FAULT,
131	#ifdef CONFIG_IOMMUFD_TEST	133	#ifdef CONFIG_IOMMUFD_TEST
132	IOMMUFD_OBJ_SELFTEST,	134	IOMMUFD_OBJ_SELFTEST,
133	#endif	135	#endif
@@ -292,6 +294,7 @@ int iommufd_check_iova_range(struct io_pagetable *iopt, · 158 unmodified lines
292	struct iommufd_hw_pagetable {	294	struct iommufd_hw_pagetable {
293	struct iommufd_object obj;	295	struct iommufd_object obj;
294	struct iommu_domain *domain;	296	struct iommu_domain *domain;
		297	struct iommufd_fault *fault;
295	};	298	};
296		299
297	struct iommufd_hwpt_paging {	300	struct iommufd_hwpt_paging {
@@ -395,6 +398,9 @@ struct iommufd_device { · 97 unmodified lines
395	/* always the physical device */	398	/* always the physical device */
396	struct device *dev;	399	struct device *dev;
397	bool enforce_cache_coherency;	400	bool enforce_cache_coherency;
		401	/* protect iopf_enabled counter */
		402	struct mutex iopf_lock;
		403	unsigned int iopf_enabled;
398	};	404	};
399		405
400	static inline struct iommufd_device *	406	static inline struct iommufd_device *
@@ -426,6 +432,80 @@ void iopt_remove_access(struct io_pagetable *iopt, · 25 unmodified lines
426	u32 iopt_access_list_id);	432	u32 iopt_access_list_id);
427	void iommufd_access_destroy_object(struct iommufd_object *obj);	433	void iommufd_access_destroy_object(struct iommufd_object *obj);
428		434
		435	/*
		436	* An iommufd_fault object represents an interface to deliver I/O page faults
		437	* to the user space. These objects are created/destroyed by the user space and
		438	* associated with hardware page table objects during page-table allocation.
		439	*/
		440	struct iommufd_fault {
		441	struct iommufd_object obj;
		442	struct iommufd_ctx *ictx;
		443	struct file *filep;
		444
		445	/* The lists of outstanding faults protected by below mutex. */
		446	struct mutex mutex;
		447	struct list_head deliver;
		448	struct xarray response;
		449
		450	struct wait_queue_head wait_queue;
		451	};
		452
		453	struct iommufd_attach_handle {
		454	struct iommu_attach_handle handle;
		455	struct iommufd_device *idev;
		456	};
		457
		458	/* Convert an iommu attach handle to iommufd handle. */
		459	#define to_iommufd_handle(hdl) container_of(hdl, struct iommufd_attach_handle, handle)
		460
		461	static inline struct iommufd_fault *
		462	iommufd_get_fault(struct iommufd_ucmd *ucmd, u32 id)
		463	{
		464	return container_of(iommufd_get_object(ucmd->ictx, id,
		465	IOMMUFD_OBJ_FAULT),
		466	struct iommufd_fault, obj);
		467	}
		468
		469	int iommufd_fault_alloc(struct iommufd_ucmd *ucmd);
		470	void iommufd_fault_destroy(struct iommufd_object *obj);
		471	int iommufd_fault_iopf_handler(struct iopf_group *group);
		472
		473	int iommufd_fault_domain_attach_dev(struct iommufd_hw_pagetable *hwpt,
		474	struct iommufd_device *idev);
		475	void iommufd_fault_domain_detach_dev(struct iommufd_hw_pagetable *hwpt,
		476	struct iommufd_device *idev);
		477	int iommufd_fault_domain_replace_dev(struct iommufd_device *idev,
		478	struct iommufd_hw_pagetable *hwpt,
		479	struct iommufd_hw_pagetable *old);
		480
		481	static inline int iommufd_hwpt_attach_device(struct iommufd_hw_pagetable *hwpt,
		482	struct iommufd_device *idev)
		483	{
		484	if (hwpt->fault)
		485	return iommufd_fault_domain_attach_dev(hwpt, idev);
		486
		487	return iommu_attach_group(hwpt->domain, idev->igroup->group);
		488	}
		489
		490	static inline void iommufd_hwpt_detach_device(struct iommufd_hw_pagetable *hwpt,
		491	struct iommufd_device *idev)
		492	{
		493	if (hwpt->fault)
		494	iommufd_fault_domain_detach_dev(hwpt, idev);
		495
		496	iommu_detach_group(hwpt->domain, idev->igroup->group);
		497	}
		498
		499	static inline int iommufd_hwpt_replace_device(struct iommufd_device *idev,
		500	struct iommufd_hw_pagetable *hwpt,
		501	struct iommufd_hw_pagetable *old)
		502	{
		503	if (old->fault \|\| hwpt->fault)
		504	return iommufd_fault_domain_replace_dev(idev, hwpt, old);
		505
		506	return iommu_group_replace_domain(idev->igroup->group, hwpt->domain);
		507	}
		508
429	#ifdef CONFIG_IOMMUFD_TEST	509	#ifdef CONFIG_IOMMUFD_TEST
430	int iommufd_test(struct iommufd_ucmd *ucmd);	510	int iommufd_test(struct iommufd_ucmd *ucmd);
431	void iommufd_selftest_destroy(struct iommufd_object *obj);	511	void iommufd_selftest_destroy(struct iommufd_object *obj);

drivers/iommu/iommufd/iommufd_test.h+8−0

 	IOMMU_TEST_OP_MOCK_DOMAIN_FLAGS,
 	IOMMU_TEST_OP_DIRTY,
 	IOMMU_TEST_OP_MD_CHECK_IOTLB,
+	IOMMU_TEST_OP_TRIGGER_IOPF,
 };
 enum {
 			__u32 id;
 			__u32 iotlb;
 		} check_iotlb;
+		struct {
+			__u32 dev_id;
+			__u32 pasid;
+			__u32 grpid;
+			__u32 perm;
+			__u64 addr;
+		} trigger_iopf;
 	};
 	__u32 last;
 };

drivers/iommu/iommufd/iova_bitmap.c+49−75

@@ -35,6 +35,9 @@ struct iova_bitmap_map { · 34 unmodified lines
35	/* base IOVA representing bit 0 of the first page */	35	/* base IOVA representing bit 0 of the first page */
36	unsigned long iova;	36	unsigned long iova;
37		37
		38	/* mapped length */
		39	unsigned long length;
		40
38	/* page size order that each bit granules to */	41	/* page size order that each bit granules to */
39	unsigned long pgshift;	42	unsigned long pgshift;
40		43
@@ -113,9 +116,6 @@ struct iova_bitmap { · 72 unmodified lines
113		116
114	/* length of the IOVA range for the whole bitmap */	117	/* length of the IOVA range for the whole bitmap */
115	size_t length;	118	size_t length;
116
117	/* length of the IOVA range set ahead the pinned pages */
118	unsigned long set_ahead_length;
119	};	119	};
120		120
121	/*	121	/*
@@ -156,6 +156,8 @@ static unsigned long iova_bitmap_mapped_iova(struct iova_bitmap *bitmap) · 34 unmodified lines
156	return bitmap->iova + iova_bitmap_index_to_offset(bitmap, skip);	156	return bitmap->iova + iova_bitmap_index_to_offset(bitmap, skip);
157	}	157	}
158		158
		159	static unsigned long iova_bitmap_mapped_length(struct iova_bitmap *bitmap);
		160
159	/*	161	/*
160	* Pins the bitmap user pages for the current range window.	162	* Pins the bitmap user pages for the current range window.
161	* This is internal to IOVA bitmap and called when advancing the	163	* This is internal to IOVA bitmap and called when advancing the
@@ -206,6 +208,7 @@ static int iova_bitmap_get(struct iova_bitmap *bitmap) · 44 unmodified lines
206	* aligned.	208	* aligned.
207	*/	209	*/
208	mapped->pgoff = offset_in_page(addr);	210	mapped->pgoff = offset_in_page(addr);
		211	mapped->length = iova_bitmap_mapped_length(bitmap);
209	return 0;	212	return 0;
210	}	213	}
211		214
@@ -263,9 +266,6 @@ struct iova_bitmap *iova_bitmap_alloc(unsigned long iova, size_t length, · 51 unmodified lines
263	goto err;	266	goto err;
264	}	267	}
265		268
266	rc = iova_bitmap_get(bitmap);
267	if (rc)
268	goto err;
269	return bitmap;	269	return bitmap;
270		270
271	err:	271	err:
@@ -338,65 +338,34 @@ static unsigned long iova_bitmap_mapped_length(struct iova_bitmap *bitmap) · 66 unmodified lines
338	}	338	}
339		339
340	/*	340	/*
341	* Returns true if there's not more data to iterate.	341	* Returns true if [@iova..@iova+@length-1] is part of the mapped IOVA range.
342	*/	342	*/
343	static bool iova_bitmap_done(struct iova_bitmap *bitmap)	343	static bool iova_bitmap_mapped_range(struct iova_bitmap_map *mapped,
		344	unsigned long iova, size_t length)
344	{	345	{
345	return bitmap->mapped_base_index >= bitmap->mapped_total_index;	346	return mapped->npages &&
346	}	347	(iova >= mapped->iova &&
347		348	(iova + length - 1) <= (mapped->iova + mapped->length - 1));
348	static int iova_bitmap_set_ahead(struct iova_bitmap *bitmap,
349	size_t set_ahead_length)
350	{
351	int ret = 0;
352
353	while (set_ahead_length > 0 && !iova_bitmap_done(bitmap)) {
354	unsigned long length = iova_bitmap_mapped_length(bitmap);
355	unsigned long iova = iova_bitmap_mapped_iova(bitmap);
356
357	ret = iova_bitmap_get(bitmap);
358	if (ret)
359	break;
360
361	length = min(length, set_ahead_length);
362	iova_bitmap_set(bitmap, iova, length);
363
364	set_ahead_length -= length;
365	bitmap->mapped_base_index +=
366	iova_bitmap_offset_to_index(bitmap, length - 1) + 1;
367	iova_bitmap_put(bitmap);
368	}
369
370	bitmap->set_ahead_length = 0;
371	return ret;
372	}	349	}
373		350
374	/*	351	/*
375	* Advances to the next range, releases the current pinned	352	* Advances to a selected range, releases the current pinned
376	* pages and pins the next set of bitmap pages.	353	* pages and pins the next set of bitmap pages.
377	* Returns 0 on success or otherwise errno.	354	* Returns 0 on success or otherwise errno.
378	*/	355	*/
379	static int iova_bitmap_advance(struct iova_bitmap *bitmap)	356	static int iova_bitmap_advance_to(struct iova_bitmap *bitmap,
		357	unsigned long iova)
380	{	358	{
381	unsigned long iova = iova_bitmap_mapped_length(bitmap) - 1;	359	unsigned long index;
382	unsigned long count = iova_bitmap_offset_to_index(bitmap, iova) + 1;
383		360
384	bitmap->mapped_base_index += count;	361	index = iova_bitmap_offset_to_index(bitmap, iova - bitmap->iova);
		362	if (index >= bitmap->mapped_total_index)
		363	return -EINVAL;
		364	bitmap->mapped_base_index = index;
385		365
386	iova_bitmap_put(bitmap);	366	iova_bitmap_put(bitmap);
387	if (iova_bitmap_done(bitmap))
388	return 0;
389
390	/* Iterate, set and skip any bits requested for next iteration */
391	if (bitmap->set_ahead_length) {
392	int ret;
393
394	ret = iova_bitmap_set_ahead(bitmap, bitmap->set_ahead_length);
395	if (ret)
396	return ret;
397	}
398		367
399	/* When advancing the index we pin the next set of bitmap pages */	368	/* Pin the next set of bitmap pages */
400	return iova_bitmap_get(bitmap);	369	return iova_bitmap_get(bitmap);
401	}	370	}
402		371
@@ -416,17 +385,7 @@ static int iova_bitmap_advance(struct iova_bitmap *bitmap) · 13 unmodified lines
416	int iova_bitmap_for_each(struct iova_bitmap bitmap, void opaque,	385	int iova_bitmap_for_each(struct iova_bitmap bitmap, void opaque,
417	iova_bitmap_fn_t fn)	386	iova_bitmap_fn_t fn)
418	{	387	{
419	int ret = 0;	388	return fn(bitmap, bitmap->iova, bitmap->length, opaque);
420
421	for (; !iova_bitmap_done(bitmap) && !ret;
422	ret = iova_bitmap_advance(bitmap)) {
423	ret = fn(bitmap, iova_bitmap_mapped_iova(bitmap),
424	iova_bitmap_mapped_length(bitmap), opaque);
425	if (ret)
426	break;
427	}
428
429	return ret;
430	}	389	}
431	EXPORT_SYMBOL_NS_GPL(iova_bitmap_for_each, IOMMUFD);	390	EXPORT_SYMBOL_NS_GPL(iova_bitmap_for_each, IOMMUFD);
432		391
@@ -444,11 +403,25 @@ void iova_bitmap_set(struct iova_bitmap *bitmap, · 11 unmodified lines
444	unsigned long iova, size_t length)	403	unsigned long iova, size_t length)
445	{	404	{
446	struct iova_bitmap_map *mapped = &bitmap->mapped;	405	struct iova_bitmap_map *mapped = &bitmap->mapped;
447	unsigned long cur_bit = ((iova - mapped->iova) >>	406	unsigned long cur_bit, last_bit, last_page_idx;
448	mapped->pgshift) + mapped->pgoff * BITS_PER_BYTE;	407
449	unsigned long last_bit = (((iova + length - 1) - mapped->iova) >>	408	update_indexes:
450	mapped->pgshift) + mapped->pgoff * BITS_PER_BYTE;	409	if (unlikely(!iova_bitmap_mapped_range(mapped, iova, length))) {
451	unsigned long last_page_idx = mapped->npages - 1;	410
		411	/*
		412	* The attempt to advance the base index to @iova
		413	* may fail if it's out of bounds, or pinning the pages
		414	* returns an error.
		415	*/
		416	if (iova_bitmap_advance_to(bitmap, iova))
		417	return;
		418	}
		419
		420	last_page_idx = mapped->npages - 1;
		421	cur_bit = ((iova - mapped->iova) >>
		422	mapped->pgshift) + mapped->pgoff * BITS_PER_BYTE;
		423	last_bit = (((iova + length - 1) - mapped->iova) >>
		424	mapped->pgshift) + mapped->pgoff * BITS_PER_BYTE;
452		425
453	do {	426	do {
454	unsigned int page_idx = cur_bit / BITS_PER_PAGE;	427	unsigned int page_idx = cur_bit / BITS_PER_PAGE;
@@ -457,18 +430,19 @@ void iova_bitmap_set(struct iova_bitmap *bitmap, · 2 unmodified lines
457	last_bit - cur_bit + 1);	430	last_bit - cur_bit + 1);
458	void *kaddr;	431	void *kaddr;
459		432
460	if (unlikely(page_idx > last_page_idx))	433	if (unlikely(page_idx > last_page_idx)) {
461	break;	434	unsigned long left =
		435	((last_bit - cur_bit + 1) << mapped->pgshift);
		436
		437	iova += (length - left);
		438	length = left;
		439	goto update_indexes;
		440	}
462		441
463	kaddr = kmap_local_page(mapped->pages[page_idx]);	442	kaddr = kmap_local_page(mapped->pages[page_idx]);
464	bitmap_set(kaddr, offset, nbits);	443	bitmap_set(kaddr, offset, nbits);
465	kunmap_local(kaddr);	444	kunmap_local(kaddr);
466	cur_bit += nbits;	445	cur_bit += nbits;
467	} while (cur_bit <= last_bit);	446	} while (cur_bit <= last_bit);
468
469	if (unlikely(cur_bit <= last_bit)) {
470	bitmap->set_ahead_length =
471	((last_bit - cur_bit + 1) << bitmap->mapped.pgshift);
472	}
473	}	447	}
474	EXPORT_SYMBOL_NS_GPL(iova_bitmap_set, IOMMUFD);	448	EXPORT_SYMBOL_NS_GPL(iova_bitmap_set, IOMMUFD);

drivers/iommu/iommufd/main.c+6−0

 union ucmd_buffer {
 	struct iommu_destroy destroy;
+	struct iommu_fault_alloc fault;
 	struct iommu_hw_info info;
 	struct iommu_hwpt_alloc hwpt;
 	struct iommu_hwpt_get_dirty_bitmap get_dirty_bitmap;
 	}
 static const struct iommufd_ioctl_op iommufd_ioctl_ops[] = {
 	IOCTL_OP(IOMMU_DESTROY, iommufd_destroy, struct iommu_destroy, id),
+	IOCTL_OP(IOMMU_FAULT_QUEUE_ALLOC, iommufd_fault_alloc, struct iommu_fault_alloc,
+		 out_fault_fd),
 	IOCTL_OP(IOMMU_GET_HW_INFO, iommufd_get_hw_info, struct iommu_hw_info,
 		 __reserved),
 	IOCTL_OP(IOMMU_HWPT_ALLOC, iommufd_hwpt_alloc, struct iommu_hwpt_alloc,
 		.destroy = iommufd_hwpt_nested_destroy,
 		.abort = iommufd_hwpt_nested_abort,
 	},
+	[IOMMUFD_OBJ_FAULT] = {
+		.destroy = iommufd_fault_destroy,
+	},
 #ifdef CONFIG_IOMMUFD_TEST
 	[IOMMUFD_OBJ_SELFTEST] = {
 		.destroy = iommufd_selftest_destroy,

drivers/iommu/iommufd/selftest.c+67−3

@@ -266,8 +266,8 @@ static int mock_domain_read_and_clear_dirty(struct iommu_domain *domain, · 265 unmodified lines
266		266
267	/* Clear dirty */	267	/* Clear dirty */
268	if (mock_test_and_clear_dirty(mock, head, pgsize, flags))	268	if (mock_test_and_clear_dirty(mock, head, pgsize, flags))
269	iommu_dirty_bitmap_record(dirty, head, pgsize);	269	iommu_dirty_bitmap_record(dirty, iova, pgsize);
270	iova = head + pgsize;	270	iova += pgsize;
271	} while (iova < end);	271	} while (iova < end);
272		272
273	return 0;	273	return 0;
@@ -504,6 +504,8 @@ static bool mock_domain_capable(struct device *dev, enum iommu_cap cap) · 230 unmodified lines
504	return false;	504	return false;
505	}	505	}
506		506
		507	static struct iopf_queue *mock_iommu_iopf_queue;
		508
507	static struct iommu_device mock_iommu_device = {	509	static struct iommu_device mock_iommu_device = {
508	};	510	};
509		511
@@ -514,6 +516,29 @@ static struct iommu_device mock_probe_device(struct device dev) · 4 unmodified lines
514	return &mock_iommu_device;	516	return &mock_iommu_device;
515	}	517	}
516		518
		519	static void mock_domain_page_response(struct device dev, struct iopf_fault evt,
		520	struct iommu_page_response *msg)
		521	{
		522	}
		523
		524	static int mock_dev_enable_feat(struct device *dev, enum iommu_dev_features feat)
		525	{
		526	if (feat != IOMMU_DEV_FEAT_IOPF \|\| !mock_iommu_iopf_queue)
		527	return -ENODEV;
		528
		529	return iopf_queue_add_device(mock_iommu_iopf_queue, dev);
		530	}
		531
		532	static int mock_dev_disable_feat(struct device *dev, enum iommu_dev_features feat)
		533	{
		534	if (feat != IOMMU_DEV_FEAT_IOPF \|\| !mock_iommu_iopf_queue)
		535	return -ENODEV;
		536
		537	iopf_queue_remove_device(mock_iommu_iopf_queue, dev);
		538
		539	return 0;
		540	}
		541
517	static const struct iommu_ops mock_ops = {	542	static const struct iommu_ops mock_ops = {
518	/*	543	/*
519	* IOMMU_DOMAIN_BLOCKED cannot be returned from def_domain_type()	544	* IOMMU_DOMAIN_BLOCKED cannot be returned from def_domain_type()
@@ -529,6 +554,10 @@ static const struct iommu_ops mock_ops = { · 9 unmodified lines
529	.capable = mock_domain_capable,	554	.capable = mock_domain_capable,
530	.device_group = generic_device_group,	555	.device_group = generic_device_group,
531	.probe_device = mock_probe_device,	556	.probe_device = mock_probe_device,
		557	.page_response = mock_domain_page_response,
		558	.dev_enable_feat = mock_dev_enable_feat,
		559	.dev_disable_feat = mock_dev_disable_feat,
		560	.user_pasid_table = true,
532	.default_domain_ops =	561	.default_domain_ops =
533	&(struct iommu_domain_ops){	562	&(struct iommu_domain_ops){
534	.free = mock_domain_free,	563	.free = mock_domain_free,
@@ -1334,7 +1363,7 @@ static int iommufd_test_dirty(struct iommufd_ucmd *ucmd, unsigned int mockpt_id, · 799 unmodified lines
1334	}	1363	}
1335		1364
1336	max = length / page_size;	1365	max = length / page_size;
1337	bitmap_size = max / BITS_PER_BYTE;	1366	bitmap_size = DIV_ROUND_UP(max, BITS_PER_BYTE);
1338		1367
1339	tmp = kvzalloc(bitmap_size, GFP_KERNEL_ACCOUNT);	1368	tmp = kvzalloc(bitmap_size, GFP_KERNEL_ACCOUNT);
1340	if (!tmp) {	1369	if (!tmp) {
@@ -1375,6 +1404,31 @@ out_put: · 34 unmodified lines
1375	return rc;	1404	return rc;
1376	}	1405	}
1377		1406
		1407	static int iommufd_test_trigger_iopf(struct iommufd_ucmd *ucmd,
		1408	struct iommu_test_cmd *cmd)
		1409	{
		1410	struct iopf_fault event = { };
		1411	struct iommufd_device *idev;
		1412
		1413	idev = iommufd_get_device(ucmd, cmd->trigger_iopf.dev_id);
		1414	if (IS_ERR(idev))
		1415	return PTR_ERR(idev);
		1416
		1417	event.fault.prm.flags = IOMMU_FAULT_PAGE_REQUEST_LAST_PAGE;
		1418	if (cmd->trigger_iopf.pasid != IOMMU_NO_PASID)
		1419	event.fault.prm.flags \|= IOMMU_FAULT_PAGE_REQUEST_PASID_VALID;
		1420	event.fault.type = IOMMU_FAULT_PAGE_REQ;
		1421	event.fault.prm.addr = cmd->trigger_iopf.addr;
		1422	event.fault.prm.pasid = cmd->trigger_iopf.pasid;
		1423	event.fault.prm.grpid = cmd->trigger_iopf.grpid;
		1424	event.fault.prm.perm = cmd->trigger_iopf.perm;
		1425
		1426	iommu_report_device_fault(idev->dev, &event);
		1427	iommufd_put_object(ucmd->ictx, &idev->obj);
		1428
		1429	return 0;
		1430	}
		1431
1378	void iommufd_selftest_destroy(struct iommufd_object *obj)	1432	void iommufd_selftest_destroy(struct iommufd_object *obj)
1379	{	1433	{
1380	struct selftest_obj *sobj = container_of(obj, struct selftest_obj, obj);	1434	struct selftest_obj *sobj = container_of(obj, struct selftest_obj, obj);
@@ -1450,6 +1504,8 @@ int iommufd_test(struct iommufd_ucmd *ucmd) · 69 unmodified lines
1450	cmd->dirty.page_size,	1504	cmd->dirty.page_size,
1451	u64_to_user_ptr(cmd->dirty.uptr),	1505	u64_to_user_ptr(cmd->dirty.uptr),
1452	cmd->dirty.flags);	1506	cmd->dirty.flags);
		1507	case IOMMU_TEST_OP_TRIGGER_IOPF:
		1508	return iommufd_test_trigger_iopf(ucmd, cmd);
1453	default:	1509	default:
1454	return -EOPNOTSUPP;	1510	return -EOPNOTSUPP;
1455	}	1511	}
@@ -1491,6 +1547,9 @@ int __init iommufd_test_init(void) · 35 unmodified lines
1491	&iommufd_mock_bus_type.nb);	1547	&iommufd_mock_bus_type.nb);
1492	if (rc)	1548	if (rc)
1493	goto err_sysfs;	1549	goto err_sysfs;
		1550
		1551	mock_iommu_iopf_queue = iopf_queue_alloc("mock-iopfq");
		1552
1494	return 0;	1553	return 0;
1495		1554
1496	err_sysfs:	1555	err_sysfs:
@@ -1506,6 +1565,11 @@ err_dbgfs: · 9 unmodified lines
1506		1565
1507	void iommufd_test_exit(void)	1566	void iommufd_test_exit(void)
1508	{	1567	{
		1568	if (mock_iommu_iopf_queue) {
		1569	iopf_queue_free(mock_iommu_iopf_queue);
		1570	mock_iommu_iopf_queue = NULL;
		1571	}
		1572
1509	iommu_device_sysfs_remove(&mock_iommu_device);	1573	iommu_device_sysfs_remove(&mock_iommu_device);
1510	iommu_device_unregister_bus(&mock_iommu_device,	1574	iommu_device_unregister_bus(&mock_iommu_device,
1511	&iommufd_mock_bus_type.bus,	1575	&iommufd_mock_bus_type.bus,

include/linux/iommu.h+25−16

 struct iopf_group {
 	struct iopf_fault last_fault;
 	struct list_head faults;
+	size_t fault_count;
 	/* list node for iommu_fault_param::faults */
 	struct list_head pending_node;
 	struct work_struct work;
-	struct iommu_domain *domain;
+	struct iommu_attach_handle *attach_handle;
 	/* The device's fault data parameter. */
 	struct iommu_fault_param *fault_param;
+	/* Used by handler provider to hook the group on its own lists. */
+	struct list_head node;
+	u32 cookie;
 };
 /**
  * @default_domain: If not NULL this will always be set as the default domain.
  *                  This should be an IDENTITY/BLOCKED/PLATFORM domain.
  *                  Do not use in new drivers.
+ * @user_pasid_table: IOMMU driver supports user-managed PASID table. There is
+ *                    no user domain for each PASID and the I/O page faults are
+ *                    forwarded through the user domain attached to the device
+ *                    RID.
  */
 struct iommu_ops {
 	bool (*capable)(struct device *dev, enum iommu_cap);
 	struct iommu_domain *blocked_domain;
 	struct iommu_domain *release_domain;
 	struct iommu_domain *default_domain;
+	u8 user_pasid_table:1;
 };
 /**
 /* ATS is supported */
 #define IOMMU_FWSPEC_PCI_RC_ATS			(1 << 0)
+/*
+ * An iommu attach handle represents a relationship between an iommu domain
+ * and a PASID or RID of a device. It is allocated and managed by the component
+ * that manages the domain and is stored in the iommu group during the time the
+ * domain is attached.
+ */
+struct iommu_attach_handle {
+	struct iommu_domain		*domain;
+};
 /**
  * struct iommu_sva - handle to a device-mm bond
  */
 struct iommu_sva {
+	struct iommu_attach_handle	handle;
 	struct device			*dev;
-	struct iommu_domain		*domain;
-	struct list_head		handle_item;
 	refcount_t			users;
 };
 struct iommu_mm_data {
 	u32			pasid;
 	struct list_head	sva_domains;
-	struct list_head	sva_handles;
 };
 int iommu_fwspec_init(struct device *dev, struct fwnode_handle *iommu_fwnode,
 void iommu_device_release_dma_owner(struct device *dev);
 int iommu_attach_device_pasid(struct iommu_domain *domain,
-			      struct device *dev, ioasid_t pasid);
+			      struct device *dev, ioasid_t pasid,
+			      struct iommu_attach_handle *handle);
 void iommu_detach_device_pasid(struct iommu_domain *domain,
 			       struct device *dev, ioasid_t pasid);
-struct iommu_domain *
-iommu_get_domain_for_dev_pasid(struct device *dev, ioasid_t pasid,
-			       unsigned int type);
 ioasid_t iommu_alloc_global_pasid(struct device *dev);
 void iommu_free_global_pasid(ioasid_t pasid);
 #else /* CONFIG_IOMMU_API */
 }
 static inline int iommu_attach_device_pasid(struct iommu_domain *domain,
-					    struct device *dev, ioasid_t pasid)
+					    struct device *dev, ioasid_t pasid,
+					    struct iommu_attach_handle *handle)
 {
 	return -ENODEV;
 }
 {
 }
-static inline struct iommu_domain *
-iommu_get_domain_for_dev_pasid(struct device *dev, ioasid_t pasid,
-			       unsigned int type)
-{
-	return NULL;
-}
 static inline ioasid_t iommu_alloc_global_pasid(struct device *dev)
 {
 	return IOMMU_PASID_INVALID;

include/uapi/linux/iommufd.h+123−18

@@ -37,19 +37,20 @@ · 36 unmodified lines
37	enum {	37	enum {
38	IOMMUFD_CMD_BASE = 0x80,	38	IOMMUFD_CMD_BASE = 0x80,
39	IOMMUFD_CMD_DESTROY = IOMMUFD_CMD_BASE,	39	IOMMUFD_CMD_DESTROY = IOMMUFD_CMD_BASE,
40	IOMMUFD_CMD_IOAS_ALLOC,	40	IOMMUFD_CMD_IOAS_ALLOC = 0x81,
41	IOMMUFD_CMD_IOAS_ALLOW_IOVAS,	41	IOMMUFD_CMD_IOAS_ALLOW_IOVAS = 0x82,
42	IOMMUFD_CMD_IOAS_COPY,	42	IOMMUFD_CMD_IOAS_COPY = 0x83,
43	IOMMUFD_CMD_IOAS_IOVA_RANGES,	43	IOMMUFD_CMD_IOAS_IOVA_RANGES = 0x84,
44	IOMMUFD_CMD_IOAS_MAP,	44	IOMMUFD_CMD_IOAS_MAP = 0x85,
45	IOMMUFD_CMD_IOAS_UNMAP,	45	IOMMUFD_CMD_IOAS_UNMAP = 0x86,
46	IOMMUFD_CMD_OPTION,	46	IOMMUFD_CMD_OPTION = 0x87,
47	IOMMUFD_CMD_VFIO_IOAS,	47	IOMMUFD_CMD_VFIO_IOAS = 0x88,
48	IOMMUFD_CMD_HWPT_ALLOC,	48	IOMMUFD_CMD_HWPT_ALLOC = 0x89,
49	IOMMUFD_CMD_GET_HW_INFO,	49	IOMMUFD_CMD_GET_HW_INFO = 0x8a,
50	IOMMUFD_CMD_HWPT_SET_DIRTY_TRACKING,	50	IOMMUFD_CMD_HWPT_SET_DIRTY_TRACKING = 0x8b,
51	IOMMUFD_CMD_HWPT_GET_DIRTY_BITMAP,	51	IOMMUFD_CMD_HWPT_GET_DIRTY_BITMAP = 0x8c,
52	IOMMUFD_CMD_HWPT_INVALIDATE,	52	IOMMUFD_CMD_HWPT_INVALIDATE = 0x8d,
		53	IOMMUFD_CMD_FAULT_QUEUE_ALLOC = 0x8e,
53	};	54	};
54		55
55	/**	56	/**
@@ -356,10 +357,13 @@ struct iommu_vfio_ioas { · 300 unmodified lines
356	* the parent HWPT in a nesting configuration.	357	* the parent HWPT in a nesting configuration.
357	* @IOMMU_HWPT_ALLOC_DIRTY_TRACKING: Dirty tracking support for device IOMMU is	358	* @IOMMU_HWPT_ALLOC_DIRTY_TRACKING: Dirty tracking support for device IOMMU is
358	* enforced on device attachment	359	* enforced on device attachment
		360	* @IOMMU_HWPT_FAULT_ID_VALID: The fault_id field of hwpt allocation data is
		361	* valid.
359	*/	362	*/
360	enum iommufd_hwpt_alloc_flags {	363	enum iommufd_hwpt_alloc_flags {
361	IOMMU_HWPT_ALLOC_NEST_PARENT = 1 << 0,	364	IOMMU_HWPT_ALLOC_NEST_PARENT = 1 << 0,
362	IOMMU_HWPT_ALLOC_DIRTY_TRACKING = 1 << 1,	365	IOMMU_HWPT_ALLOC_DIRTY_TRACKING = 1 << 1,
		366	IOMMU_HWPT_FAULT_ID_VALID = 1 << 2,
363	};	367	};
364		368
365	/**	369	/**
@@ -396,8 +400,8 @@ struct iommu_hwpt_vtd_s1 { · 30 unmodified lines
396	* @IOMMU_HWPT_DATA_VTD_S1: Intel VT-d stage-1 page table	400	* @IOMMU_HWPT_DATA_VTD_S1: Intel VT-d stage-1 page table
397	*/	401	*/
398	enum iommu_hwpt_data_type {	402	enum iommu_hwpt_data_type {
399	IOMMU_HWPT_DATA_NONE,	403	IOMMU_HWPT_DATA_NONE = 0,
400	IOMMU_HWPT_DATA_VTD_S1,	404	IOMMU_HWPT_DATA_VTD_S1 = 1,
401	};	405	};
402		406
403	/**	407	/**
@@ -411,6 +415,9 @@ enum iommu_hwpt_data_type { · 7 unmodified lines
411	* @data_type: One of enum iommu_hwpt_data_type	415	* @data_type: One of enum iommu_hwpt_data_type
412	* @data_len: Length of the type specific data	416	* @data_len: Length of the type specific data
413	* @data_uptr: User pointer to the type specific data	417	* @data_uptr: User pointer to the type specific data
		418	* @fault_id: The ID of IOMMUFD_FAULT object. Valid only if flags field of
		419	* IOMMU_HWPT_FAULT_ID_VALID is set.
		420	* @__reserved2: Padding to 64-bit alignment. Must be 0.
414	*	421	*
415	* Explicitly allocate a hardware page table object. This is the same object	422	* Explicitly allocate a hardware page table object. This is the same object
416	* type that is returned by iommufd_device_attach() and represents the	423	* type that is returned by iommufd_device_attach() and represents the
@@ -441,6 +448,8 @@ struct iommu_hwpt_alloc { · 24 unmodified lines
441	__u32 data_type;	448	__u32 data_type;
442	__u32 data_len;	449	__u32 data_len;
443	__aligned_u64 data_uptr;	450	__aligned_u64 data_uptr;
		451	__u32 fault_id;
		452	__u32 __reserved2;
444	};	453	};
445	#define IOMMU_HWPT_ALLOC _IO(IOMMUFD_TYPE, IOMMUFD_CMD_HWPT_ALLOC)	454	#define IOMMU_HWPT_ALLOC _IO(IOMMUFD_TYPE, IOMMUFD_CMD_HWPT_ALLOC)
446		455
@@ -482,8 +491,8 @@ struct iommu_hw_info_vtd { · 35 unmodified lines
482	* @IOMMU_HW_INFO_TYPE_INTEL_VTD: Intel VT-d iommu info type	491	* @IOMMU_HW_INFO_TYPE_INTEL_VTD: Intel VT-d iommu info type
483	*/	492	*/
484	enum iommu_hw_info_type {	493	enum iommu_hw_info_type {
485	IOMMU_HW_INFO_TYPE_NONE,	494	IOMMU_HW_INFO_TYPE_NONE = 0,
486	IOMMU_HW_INFO_TYPE_INTEL_VTD,	495	IOMMU_HW_INFO_TYPE_INTEL_VTD = 1,
487	};	496	};
488		497
489	/**	498	/**
@@ -620,7 +629,7 @@ struct iommu_hwpt_get_dirty_bitmap { · 130 unmodified lines
620	* @IOMMU_HWPT_INVALIDATE_DATA_VTD_S1: Invalidation data for VTD_S1	629	* @IOMMU_HWPT_INVALIDATE_DATA_VTD_S1: Invalidation data for VTD_S1
621	*/	630	*/
622	enum iommu_hwpt_invalidate_data_type {	631	enum iommu_hwpt_invalidate_data_type {
623	IOMMU_HWPT_INVALIDATE_DATA_VTD_S1,	632	IOMMU_HWPT_INVALIDATE_DATA_VTD_S1 = 0,
624	};	633	};
625		634
626	/**	635	/**
@@ -692,4 +701,100 @@ struct iommu_hwpt_invalidate { · 65 unmodified lines
692	__u32 __reserved;	701	__u32 __reserved;
693	};	702	};
694	#define IOMMU_HWPT_INVALIDATE _IO(IOMMUFD_TYPE, IOMMUFD_CMD_HWPT_INVALIDATE)	703	#define IOMMU_HWPT_INVALIDATE _IO(IOMMUFD_TYPE, IOMMUFD_CMD_HWPT_INVALIDATE)
		704
		705	/**
		706	* enum iommu_hwpt_pgfault_flags - flags for struct iommu_hwpt_pgfault
		707	* @IOMMU_PGFAULT_FLAGS_PASID_VALID: The pasid field of the fault data is
		708	* valid.
		709	* @IOMMU_PGFAULT_FLAGS_LAST_PAGE: It's the last fault of a fault group.
		710	*/
		711	enum iommu_hwpt_pgfault_flags {
		712	IOMMU_PGFAULT_FLAGS_PASID_VALID = (1 << 0),
		713	IOMMU_PGFAULT_FLAGS_LAST_PAGE = (1 << 1),
		714	};
		715
		716	/**
		717	* enum iommu_hwpt_pgfault_perm - perm bits for struct iommu_hwpt_pgfault
		718	* @IOMMU_PGFAULT_PERM_READ: request for read permission
		719	* @IOMMU_PGFAULT_PERM_WRITE: request for write permission
		720	* @IOMMU_PGFAULT_PERM_EXEC: (PCIE 10.4.1) request with a PASID that has the
		721	* Execute Requested bit set in PASID TLP Prefix.
		722	* @IOMMU_PGFAULT_PERM_PRIV: (PCIE 10.4.1) request with a PASID that has the
		723	* Privileged Mode Requested bit set in PASID TLP
		724	* Prefix.
		725	*/
		726	enum iommu_hwpt_pgfault_perm {
		727	IOMMU_PGFAULT_PERM_READ = (1 << 0),
		728	IOMMU_PGFAULT_PERM_WRITE = (1 << 1),
		729	IOMMU_PGFAULT_PERM_EXEC = (1 << 2),
		730	IOMMU_PGFAULT_PERM_PRIV = (1 << 3),
		731	};
		732
		733	/**
		734	* struct iommu_hwpt_pgfault - iommu page fault data
		735	* @flags: Combination of enum iommu_hwpt_pgfault_flags
		736	* @dev_id: id of the originated device
		737	* @pasid: Process Address Space ID
		738	* @grpid: Page Request Group Index
		739	* @perm: Combination of enum iommu_hwpt_pgfault_perm
		740	* @addr: Fault address
		741	* @length: a hint of how much data the requestor is expecting to fetch. For
		742	* example, if the PRI initiator knows it is going to do a 10MB
		743	* transfer, it could fill in 10MB and the OS could pre-fault in
		744	* 10MB of IOVA. It's default to 0 if there's no such hint.
		745	* @cookie: kernel-managed cookie identifying a group of fault messages. The
		746	* cookie number encoded in the last page fault of the group should
		747	* be echoed back in the response message.
		748	*/
		749	struct iommu_hwpt_pgfault {
		750	__u32 flags;
		751	__u32 dev_id;
		752	__u32 pasid;
		753	__u32 grpid;
		754	__u32 perm;
		755	__u64 addr;
		756	__u32 length;
		757	__u32 cookie;
		758	};
		759
		760	/**
		761	* enum iommufd_page_response_code - Return status of fault handlers
		762	* @IOMMUFD_PAGE_RESP_SUCCESS: Fault has been handled and the page tables
		763	* populated, retry the access. This is the
		764	* "Success" defined in PCI 10.4.2.1.
		765	* @IOMMUFD_PAGE_RESP_INVALID: Could not handle this fault, don't retry the
		766	* access. This is the "Invalid Request" in PCI
		767	* 10.4.2.1.
		768	*/
		769	enum iommufd_page_response_code {
		770	IOMMUFD_PAGE_RESP_SUCCESS = 0,
		771	IOMMUFD_PAGE_RESP_INVALID = 1,
		772	};
		773
		774	/**
		775	* struct iommu_hwpt_page_response - IOMMU page fault response
		776	* @cookie: The kernel-managed cookie reported in the fault message.
		777	* @code: One of response code in enum iommufd_page_response_code.
		778	*/
		779	struct iommu_hwpt_page_response {
		780	__u32 cookie;
		781	__u32 code;
		782	};
		783
		784	/**
		785	* struct iommu_fault_alloc - ioctl(IOMMU_FAULT_QUEUE_ALLOC)
		786	* @size: sizeof(struct iommu_fault_alloc)
		787	* @flags: Must be 0
		788	* @out_fault_id: The ID of the new FAULT
		789	* @out_fault_fd: The fd of the new FAULT
		790	*
		791	* Explicitly allocate a fault handling object.
		792	*/
		793	struct iommu_fault_alloc {
		794	__u32 size;
		795	__u32 flags;
		796	__u32 out_fault_id;
		797	__u32 out_fault_fd;
		798	};
		799	#define IOMMU_FAULT_QUEUE_ALLOC _IO(IOMMUFD_TYPE, IOMMUFD_CMD_FAULT_QUEUE_ALLOC)
695	#endif	800	#endif

tools/testing/selftests/iommu/iommufd.c+63−23

 	uint32_t parent_hwpt_id = 0;
 	uint32_t parent_hwpt_id_not_work = 0;
 	uint32_t test_hwpt_id = 0;
+	uint32_t iopf_hwpt_id;
+	uint32_t fault_id;
+	uint32_t fault_fd;
 	if (self->device_id) {
 		/* Negative tests */
 					   sizeof(data));
 		/* Allocate two nested hwpts sharing one common parent hwpt */
+		test_ioctl_fault_alloc(&fault_id, &fault_fd);
 		test_cmd_hwpt_alloc_nested(self->device_id, parent_hwpt_id, 0,
 					   &nested_hwpt_id[0],
 					   IOMMU_HWPT_DATA_SELFTEST, &data,
 					   &nested_hwpt_id[1],
 					   IOMMU_HWPT_DATA_SELFTEST, &data,
 					   sizeof(data));
+		test_err_hwpt_alloc_iopf(ENOENT, self->device_id, parent_hwpt_id,
+					 UINT32_MAX, IOMMU_HWPT_FAULT_ID_VALID,
+					 &iopf_hwpt_id, IOMMU_HWPT_DATA_SELFTEST,
+					 &data, sizeof(data));
+		test_cmd_hwpt_alloc_iopf(self->device_id, parent_hwpt_id, fault_id,
+					 IOMMU_HWPT_FAULT_ID_VALID, &iopf_hwpt_id,
+					 IOMMU_HWPT_DATA_SELFTEST, &data,
+					 sizeof(data));
 		test_cmd_hwpt_check_iotlb_all(nested_hwpt_id[0],
 					      IOMMU_TEST_IOTLB_DEFAULT);
 		test_cmd_hwpt_check_iotlb_all(nested_hwpt_id[1],
 			     _test_ioctl_destroy(self->fd, nested_hwpt_id[1]));
 		test_ioctl_destroy(nested_hwpt_id[0]);
+		/* Switch from nested_hwpt_id[1] to iopf_hwpt_id */
+		test_cmd_mock_domain_replace(self->stdev_id, iopf_hwpt_id);
+		EXPECT_ERRNO(EBUSY,
+			     _test_ioctl_destroy(self->fd, iopf_hwpt_id));
+		/* Trigger an IOPF on the device */
+		test_cmd_trigger_iopf(self->device_id, fault_fd);
 		/* Detach from nested_hwpt_id[1] and destroy it */
 		test_cmd_mock_domain_replace(self->stdev_id, parent_hwpt_id);
 		test_ioctl_destroy(nested_hwpt_id[1]);
+		test_ioctl_destroy(iopf_hwpt_id);
 		/* Detach from the parent hw_pagetable and destroy it */
 		test_cmd_mock_domain_replace(self->stdev_id, self->ioas_id);
 		test_ioctl_destroy(parent_hwpt_id);
 		test_ioctl_destroy(parent_hwpt_id_not_work);
+		close(fault_fd);
+		test_ioctl_destroy(fault_id);
 	} else {
 		test_err_hwpt_alloc(ENOENT, self->device_id, self->ioas_id, 0,
 				    &parent_hwpt_id);
 FIXTURE_SETUP(iommufd_dirty_tracking)
 {
+	unsigned long size;
 	int mmap_flags;
 	void *vrc;
 	int rc;
+	if (variant->buffer_size < MOCK_PAGE_SIZE) {
+		SKIP(return,
+		     "Skipping buffer_size=%lu, less than MOCK_PAGE_SIZE=%lu",
+		     variant->buffer_size, MOCK_PAGE_SIZE);
+	}
 	self->fd = open("/dev/iommu", O_RDWR);
 	ASSERT_NE(-1, self->fd);
 	assert(vrc == self->buffer);
 	self->page_size = MOCK_PAGE_SIZE;
-	self->bitmap_size =
+	self->bitmap_size = variant->buffer_size / self->page_size;
-		variant->buffer_size / self->page_size / BITS_PER_BYTE;
 	/* Provision with an extra (PAGE_SIZE) for the unaligned case */
-	rc = posix_memalign(&self->bitmap, PAGE_SIZE,
+	size = DIV_ROUND_UP(self->bitmap_size, BITS_PER_BYTE);
-			    self->bitmap_size + PAGE_SIZE);
+	rc = posix_memalign(&self->bitmap, PAGE_SIZE, size + PAGE_SIZE);
 	assert(!rc);
 	assert(self->bitmap);
 	assert((uintptr_t)self->bitmap % PAGE_SIZE == 0);
 FIXTURE_TEARDOWN(iommufd_dirty_tracking)
 {
 	munmap(self->buffer, variant->buffer_size);
-	munmap(self->bitmap, self->bitmap_size);
+	munmap(self->bitmap, DIV_ROUND_UP(self->bitmap_size, BITS_PER_BYTE));
 	teardown_iommufd(self->fd, _metadata);
 }
-FIXTURE_VARIANT_ADD(iommufd_dirty_tracking, domain_dirty128k)
+FIXTURE_VARIANT_ADD(iommufd_dirty_tracking, domain_dirty8k)
+{
+	/* half of an u8 index bitmap */
+	.buffer_size = 8UL * 1024UL,
+};
+FIXTURE_VARIANT_ADD(iommufd_dirty_tracking, domain_dirty16k)
+{
+	/* one u8 index bitmap */
+	.buffer_size = 16UL * 1024UL,
+};
+FIXTURE_VARIANT_ADD(iommufd_dirty_tracking, domain_dirty64k)
 {
 	/* one u32 index bitmap */
-	.buffer_size = 128UL * 1024UL,
+	.buffer_size = 64UL * 1024UL,
 };
-FIXTURE_VARIANT_ADD(iommufd_dirty_tracking, domain_dirty256k)
+FIXTURE_VARIANT_ADD(iommufd_dirty_tracking, domain_dirty128k)
 {
 	/* one u64 index bitmap */
-	.buffer_size = 256UL * 1024UL,
+	.buffer_size = 128UL * 1024UL,
 };
-FIXTURE_VARIANT_ADD(iommufd_dirty_tracking, domain_dirty640k)
+FIXTURE_VARIANT_ADD(iommufd_dirty_tracking, domain_dirty320k)
 {
 	/* two u64 index and trailing end bitmap */
-	.buffer_size = 640UL * 1024UL,
+	.buffer_size = 320UL * 1024UL,
 };
-FIXTURE_VARIANT_ADD(iommufd_dirty_tracking, domain_dirty128M)
+FIXTURE_VARIANT_ADD(iommufd_dirty_tracking, domain_dirty64M)
 {
-	/* 4K bitmap (128M IOVA range) */
+	/* 4K bitmap (64M IOVA range) */
-	.buffer_size = 128UL * 1024UL * 1024UL,
+	.buffer_size = 64UL * 1024UL * 1024UL,
 };
-FIXTURE_VARIANT_ADD(iommufd_dirty_tracking, domain_dirty128M_huge)
+FIXTURE_VARIANT_ADD(iommufd_dirty_tracking, domain_dirty64M_huge)
 {
-	/* 4K bitmap (128M IOVA range) */
+	/* 4K bitmap (64M IOVA range) */
-	.buffer_size = 128UL * 1024UL * 1024UL,
+	.buffer_size = 64UL * 1024UL * 1024UL,
 	.hugepages = true,
 };
-FIXTURE_VARIANT_ADD(iommufd_dirty_tracking, domain_dirty256M)
+FIXTURE_VARIANT_ADD(iommufd_dirty_tracking, domain_dirty128M)
 {
-	/* 8K bitmap (256M IOVA range) */
+	/* 8K bitmap (128M IOVA range) */
-	.buffer_size = 256UL * 1024UL * 1024UL,
+	.buffer_size = 128UL * 1024UL * 1024UL,
 };
-FIXTURE_VARIANT_ADD(iommufd_dirty_tracking, domain_dirty256M_huge)
+FIXTURE_VARIANT_ADD(iommufd_dirty_tracking, domain_dirty128M_huge)
 {
-	/* 8K bitmap (256M IOVA range) */
+	/* 8K bitmap (128M IOVA range) */
-	.buffer_size = 256UL * 1024UL * 1024UL,
+	.buffer_size = 128UL * 1024UL * 1024UL,
 	.hugepages = true,
 };

tools/testing/selftests/iommu/iommufd_fail_nth.c+1−1

@@ -615,7 +615,7 @@ TEST_FAIL_NTH(basic_fail_nth, device) · 614 unmodified lines
615	if (_test_cmd_get_hw_info(self->fd, idev_id, &info, sizeof(info), NULL))	615	if (_test_cmd_get_hw_info(self->fd, idev_id, &info, sizeof(info), NULL))
616	return -1;	616	return -1;
617		617
618	if (_test_cmd_hwpt_alloc(self->fd, idev_id, ioas_id, 0, &hwpt_id,	618	if (_test_cmd_hwpt_alloc(self->fd, idev_id, ioas_id, 0, 0, &hwpt_id,
619	IOMMU_HWPT_DATA_NONE, 0, 0))	619	IOMMU_HWPT_DATA_NONE, 0, 0))
620	return -1;	620	return -1;
621		621

tools/testing/selftests/iommu/iommufd_utils.h+85−7

@@ -22,6 +22,8 @@ · 21 unmodified lines
22	#define BIT_MASK(nr) (1UL << ((nr) % __BITS_PER_LONG))	22	#define BIT_MASK(nr) (1UL << ((nr) % __BITS_PER_LONG))
23	#define BIT_WORD(nr) ((nr) / __BITS_PER_LONG)	23	#define BIT_WORD(nr) ((nr) / __BITS_PER_LONG)
24		24
		25	#define DIV_ROUND_UP(n, d) (((n) + (d) - 1) / (d))
		26
25	static inline void set_bit(unsigned int nr, unsigned long *addr)	27	static inline void set_bit(unsigned int nr, unsigned long *addr)
26	{	28	{
27	unsigned long mask = BIT_MASK(nr);	29	unsigned long mask = BIT_MASK(nr);
@@ -153,7 +155,7 @@ static int _test_cmd_mock_domain_replace(int fd, __u32 stdev_id, __u32 pt_id, · 125 unmodified lines
153	EXPECT_ERRNO(_errno, _test_cmd_mock_domain_replace(self->fd, stdev_id, \	155	EXPECT_ERRNO(_errno, _test_cmd_mock_domain_replace(self->fd, stdev_id, \
154	pt_id, NULL))	156	pt_id, NULL))
155		157
156	static int _test_cmd_hwpt_alloc(int fd, __u32 device_id, __u32 pt_id,	158	static int _test_cmd_hwpt_alloc(int fd, __u32 device_id, __u32 pt_id, __u32 ft_id,
157	__u32 flags, __u32 *hwpt_id, __u32 data_type,	159	__u32 flags, __u32 *hwpt_id, __u32 data_type,
158	void *data, size_t data_len)	160	void *data, size_t data_len)
159	{	161	{
@@ -165,6 +167,7 @@ static int _test_cmd_hwpt_alloc(int fd, __u32 device_id, __u32 pt_id, · 5 unmodified lines
165	.data_type = data_type,	167	.data_type = data_type,
166	.data_len = data_len,	168	.data_len = data_len,
167	.data_uptr = (uint64_t)data,	169	.data_uptr = (uint64_t)data,
		170	.fault_id = ft_id,
168	};	171	};
169	int ret;	172	int ret;
170		173
@@ -177,24 +180,36 @@ static int _test_cmd_hwpt_alloc(int fd, __u32 device_id, __u32 pt_id, · 6 unmodified lines
177	}	180	}
178		181
179	#define test_cmd_hwpt_alloc(device_id, pt_id, flags, hwpt_id) \	182	#define test_cmd_hwpt_alloc(device_id, pt_id, flags, hwpt_id) \
180	ASSERT_EQ(0, _test_cmd_hwpt_alloc(self->fd, device_id, pt_id, flags, \	183	ASSERT_EQ(0, _test_cmd_hwpt_alloc(self->fd, device_id, pt_id, 0, flags, \
181	hwpt_id, IOMMU_HWPT_DATA_NONE, NULL, \	184	hwpt_id, IOMMU_HWPT_DATA_NONE, NULL, \
182	0))	185	0))
183	#define test_err_hwpt_alloc(_errno, device_id, pt_id, flags, hwpt_id) \	186	#define test_err_hwpt_alloc(_errno, device_id, pt_id, flags, hwpt_id) \
184	EXPECT_ERRNO(_errno, _test_cmd_hwpt_alloc( \	187	EXPECT_ERRNO(_errno, _test_cmd_hwpt_alloc( \
185	self->fd, device_id, pt_id, flags, \	188	self->fd, device_id, pt_id, 0, flags, \
186	hwpt_id, IOMMU_HWPT_DATA_NONE, NULL, 0))	189	hwpt_id, IOMMU_HWPT_DATA_NONE, NULL, 0))
187		190
188	#define test_cmd_hwpt_alloc_nested(device_id, pt_id, flags, hwpt_id, \	191	#define test_cmd_hwpt_alloc_nested(device_id, pt_id, flags, hwpt_id, \
189	data_type, data, data_len) \	192	data_type, data, data_len) \
190	ASSERT_EQ(0, _test_cmd_hwpt_alloc(self->fd, device_id, pt_id, flags, \	193	ASSERT_EQ(0, _test_cmd_hwpt_alloc(self->fd, device_id, pt_id, 0, flags, \
191	hwpt_id, data_type, data, data_len))	194	hwpt_id, data_type, data, data_len))
192	#define test_err_hwpt_alloc_nested(_errno, device_id, pt_id, flags, hwpt_id, \	195	#define test_err_hwpt_alloc_nested(_errno, device_id, pt_id, flags, hwpt_id, \
193	data_type, data, data_len) \	196	data_type, data, data_len) \
194	EXPECT_ERRNO(_errno, \	197	EXPECT_ERRNO(_errno, \
195	_test_cmd_hwpt_alloc(self->fd, device_id, pt_id, flags, \	198	_test_cmd_hwpt_alloc(self->fd, device_id, pt_id, 0, flags, \
196	hwpt_id, data_type, data, data_len))	199	hwpt_id, data_type, data, data_len))
197		200
		201	#define test_cmd_hwpt_alloc_iopf(device_id, pt_id, fault_id, flags, hwpt_id, \
		202	data_type, data, data_len) \
		203	ASSERT_EQ(0, _test_cmd_hwpt_alloc(self->fd, device_id, pt_id, fault_id, \
		204	flags, hwpt_id, data_type, data, \
		205	data_len))
		206	#define test_err_hwpt_alloc_iopf(_errno, device_id, pt_id, fault_id, flags, \
		207	hwpt_id, data_type, data, data_len) \
		208	EXPECT_ERRNO(_errno, \
		209	_test_cmd_hwpt_alloc(self->fd, device_id, pt_id, fault_id, \
		210	flags, hwpt_id, data_type, data, \
		211	data_len))
		212
198	#define test_cmd_hwpt_check_iotlb(hwpt_id, iotlb_id, expected) \	213	#define test_cmd_hwpt_check_iotlb(hwpt_id, iotlb_id, expected) \
199	({ \	214	({ \
200	struct iommu_test_cmd test_cmd = { \	215	struct iommu_test_cmd test_cmd = { \
@@ -346,12 +361,12 @@ static int _test_cmd_mock_domain_set_dirty(int fd, __u32 hwpt_id, size_t length, · 145 unmodified lines
346	static int _test_mock_dirty_bitmaps(int fd, __u32 hwpt_id, size_t length,	361	static int _test_mock_dirty_bitmaps(int fd, __u32 hwpt_id, size_t length,
347	__u64 iova, size_t page_size,	362	__u64 iova, size_t page_size,
348	size_t pte_page_size, __u64 *bitmap,	363	size_t pte_page_size, __u64 *bitmap,
349	__u64 bitmap_size, __u32 flags,	364	__u64 nbits, __u32 flags,
350	struct __test_metadata *_metadata)	365	struct __test_metadata *_metadata)
351	{	366	{
352	unsigned long npte = pte_page_size / page_size, pteset = 2 * npte;	367	unsigned long npte = pte_page_size / page_size, pteset = 2 * npte;
353	unsigned long nbits = bitmap_size * BITS_PER_BYTE;
354	unsigned long j, i, nr = nbits / pteset ?: 1;	368	unsigned long j, i, nr = nbits / pteset ?: 1;
		369	unsigned long bitmap_size = DIV_ROUND_UP(nbits, BITS_PER_BYTE);
355	__u64 out_dirty = 0;	370	__u64 out_dirty = 0;
356		371
357	/* Mark all even bits as dirty in the mock domain */	372	/* Mark all even bits as dirty in the mock domain */
@@ -684,3 +699,66 @@ static int _test_cmd_get_hw_info(int fd, __u32 device_id, void *data, · 326 unmodified lines
684		699
685	#define test_cmd_get_hw_capabilities(device_id, caps, mask) \	700	#define test_cmd_get_hw_capabilities(device_id, caps, mask) \
686	ASSERT_EQ(0, _test_cmd_get_hw_info(self->fd, device_id, NULL, 0, &caps))	701	ASSERT_EQ(0, _test_cmd_get_hw_info(self->fd, device_id, NULL, 0, &caps))
		702
		703	static int _test_ioctl_fault_alloc(int fd, __u32 fault_id, __u32 fault_fd)
		704	{
		705	struct iommu_fault_alloc cmd = {
		706	.size = sizeof(cmd),
		707	};
		708	int ret;
		709
		710	ret = ioctl(fd, IOMMU_FAULT_QUEUE_ALLOC, &cmd);
		711	if (ret)
		712	return ret;
		713	*fault_id = cmd.out_fault_id;
		714	*fault_fd = cmd.out_fault_fd;
		715	return 0;
		716	}
		717
		718	#define test_ioctl_fault_alloc(fault_id, fault_fd) \
		719	({ \
		720	ASSERT_EQ(0, _test_ioctl_fault_alloc(self->fd, fault_id, \
		721	fault_fd)); \
		722	ASSERT_NE(0, *(fault_id)); \
		723	ASSERT_NE(0, *(fault_fd)); \
		724	})
		725
		726	static int _test_cmd_trigger_iopf(int fd, __u32 device_id, __u32 fault_fd)
		727	{
		728	struct iommu_test_cmd trigger_iopf_cmd = {
		729	.size = sizeof(trigger_iopf_cmd),
		730	.op = IOMMU_TEST_OP_TRIGGER_IOPF,
		731	.trigger_iopf = {
		732	.dev_id = device_id,
		733	.pasid = 0x1,
		734	.grpid = 0x2,
		735	.perm = IOMMU_PGFAULT_PERM_READ \| IOMMU_PGFAULT_PERM_WRITE,
		736	.addr = 0xdeadbeaf,
		737	},
		738	};
		739	struct iommu_hwpt_page_response response = {
		740	.code = IOMMUFD_PAGE_RESP_SUCCESS,
		741	};
		742	struct iommu_hwpt_pgfault fault = {};
		743	ssize_t bytes;
		744	int ret;
		745
		746	ret = ioctl(fd, _IOMMU_TEST_CMD(IOMMU_TEST_OP_TRIGGER_IOPF), &trigger_iopf_cmd);
		747	if (ret)
		748	return ret;
		749
		750	bytes = read(fault_fd, &fault, sizeof(fault));
		751	if (bytes <= 0)
		752	return -EIO;
		753
		754	response.cookie = fault.cookie;
		755
		756	bytes = write(fault_fd, &response, sizeof(response));
		757	if (bytes <= 0)
		758	return -EIO;
		759
		760	return 0;
		761	}
		762
		763	#define test_cmd_trigger_iopf(device_id, fault_fd) \
		764	ASSERT_EQ(0, _test_cmd_trigger_iopf(self->fd, device_id, fault_fd))