Merge tag 'for-linus-iommufd' of git://git.kernel.org/pub/scm/linux/kernel/git/jgg/iommufd

Pull iommufd updates from Jason Gunthorpe: - The iova_bitmap logic for efficiently reporting dirty pages back to userspace has a few more tricky corner case bugs that have been resolved and backed with new tests. The revised version has simpler logic. - Shared branch with iommu for handle support when doing domain attach. Handles allow the domain owner to include additional private data on a per-device basis. - IO Page Fault Reporting to userspace via iommufd. Page faults can be generated on fault capable HWPTs when a translation is not present. Routing them to userspace would allow a VMM to be able to virtualize them into an emulated vIOMMU. This is the next step to fully enabling vSVA support. * tag 'for-linus-iommufd' of git://git.kernel.org/pub/scm/linux/kernel/git/jgg/iommufd: (26 commits) iommufd: Put constants for all the uAPI enums iommufd: Fix error pointer checking iommufd: Add check on user response code iommufd: Remove IOMMUFD_PAGE_RESP_FAILURE iommufd: Require drivers to supply the cache_invalidate_user ops iommufd/selftest: Add coverage for IOPF test iommufd/selftest: Add IOPF support for mock device iommufd: Associate fault object with iommufd_hw_pgtable iommufd: Fault-capable hwpt attach/detach/replace iommufd: Add iommufd fault object iommufd: Add fault and response message definitions iommu: Extend domain attach group with handle support iommu: Add attach handle to struct iopf_group iommu: Remove sva handle list iommu: Introduce domain attachment handle iommufd/iova_bitmap: Remove iterator logic iommufd/iova_bitmap: Dynamic pinning on iova_bitmap_set() iommufd/iova_bitmap: Consolidate iova_bitmap_set exit conditionals iommufd/iova_bitmap: Move initial pinning to iova_bitmap_for_each() iommufd/iova_bitmap: Cache mapped length in iova_bitmap_map struct ...

Linus Torvalds committed 2 years ago

Commit ef7c8f22 parents 07e773d 136a80619 files · +1206 −239Browse files

Unified · Split

19 files

drivers/dma/idxd/init.c +1 −1
drivers/iommu/io-pgfault.c +36 −27
drivers/iommu/iommu-priv.h +11 −0
drivers/iommu/iommu-sva.c +28 −14
drivers/iommu/iommu.c +143 −42
drivers/iommu/iommufd/Makefile +1 −0
drivers/iommu/iommufd/device.c +4 −3
drivers/iommu/iommufd/fault.c +443 −0
drivers/iommu/iommufd/hw_pagetable.c +32 −9
drivers/iommu/iommufd/iommufd_private.h +80 −0
drivers/iommu/iommufd/iommufd_test.h +8 −0
drivers/iommu/iommufd/iova_bitmap.c +49 −75
drivers/iommu/iommufd/main.c +6 −0
drivers/iommu/iommufd/selftest.c +67 −3
include/linux/iommu.h +25 −16
include/uapi/linux/iommufd.h +123 −18
tools/testing/selftests/iommu/iommufd.c +63 −23
tools/testing/selftests/iommu/iommufd_fail_nth.c +1 −1
tools/testing/selftests/iommu/iommufd_utils.h +85 −7

drivers/dma/idxd/init.c+1−1

@@ -584,7 +584,7 @@ static int idxd_enable_system_pasid(struct idxd_device *idxd) · 583 unmodified lines
584	584	* DMA domain is owned by the driver, it should support all valid
585	585	* types such as DMA-FQ, identity, etc.
586	586	*/
587		− ret = iommu_attach_device_pasid(domain, dev, pasid);
	587	+ ret = iommu_attach_device_pasid(domain, dev, pasid, NULL);
588	588	if (ret) {
589	589	dev_err(dev, "failed to attach device pasid %d, domain type %d",
590	590	pasid, domain->type);

drivers/iommu/io-pgfault.c+36−27

@@ -59,30 +59,6 @@ void iopf_free_group(struct iopf_group *group) · 58 unmodified lines
59	59	}
60	60	EXPORT_SYMBOL_GPL(iopf_free_group);
61	61
62		−static struct iommu_domain get_domain_for_iopf(struct device dev,
63		− struct iommu_fault *fault)
64		−{
65		− struct iommu_domain *domain;
66		−
67		− if (fault->prm.flags & IOMMU_FAULT_PAGE_REQUEST_PASID_VALID) {
68		− domain = iommu_get_domain_for_dev_pasid(dev, fault->prm.pasid, 0);
69		− if (IS_ERR(domain))
70		− domain = NULL;
71		− } else {
72		− domain = iommu_get_domain_for_dev(dev);
73		− }
74		−
75		− if (!domain \|\| !domain->iopf_handler) {
76		− dev_warn_ratelimited(dev,
77		− "iopf (pasid %d) without domain attached or handler installed\n",
78		− fault->prm.pasid);
79		−
80		− return NULL;
81		− }
82		−
83		− return domain;
84		−}
85		−
86	62	/* Non-last request of a group. Postpone until the last one. */
87	63	static int report_partial_fault(struct iommu_fault_param *fault_param,
88	64	struct iommu_fault *fault)
@@ -134,6 +110,8 @@ static struct iopf_group iopf_group_alloc(struct iommu_fault_param iopf_param, · 45 unmodified lines
134	110	list_add(&group->pending_node, &iopf_param->faults);
135	111	mutex_unlock(&iopf_param->lock);
136	112
	113	+ group->fault_count = list_count_nodes(&group->faults);
	114	+
137	115	return group;
138	116	}
139	117
@@ -206,20 +184,51 @@ void iommu_report_device_fault(struct device dev, struct iopf_fault evt) · 66 unmodified lines
206	184	if (group == &abort_group)
207	185	goto err_abort;
208	186
209		− group->domain = get_domain_for_iopf(dev, fault);
210		− if (!group->domain)
	187	+ if (fault->prm.flags & IOMMU_FAULT_PAGE_REQUEST_PASID_VALID) {
	188	+ group->attach_handle = iommu_attach_handle_get(dev->iommu_group,
	189	+ fault->prm.pasid,
	190	+ 0);
	191	+ if (IS_ERR(group->attach_handle)) {
	192	+ const struct iommu_ops *ops = dev_iommu_ops(dev);
	193	+
	194	+ if (!ops->user_pasid_table)
	195	+ goto err_abort;
	196	+
	197	+ /*
	198	+ * The iommu driver for this device supports user-
	199	+ * managed PASID table. Therefore page faults for
	200	+ * any PASID should go through the NESTING domain
	201	+ * attached to the device RID.
	202	+ */
	203	+ group->attach_handle =
	204	+ iommu_attach_handle_get(dev->iommu_group,
	205	+ IOMMU_NO_PASID,
	206	+ IOMMU_DOMAIN_NESTED);
	207	+ if (IS_ERR(group->attach_handle))
	208	+ goto err_abort;
	209	+ }
	210	+ } else {
	211	+ group->attach_handle =
	212	+ iommu_attach_handle_get(dev->iommu_group, IOMMU_NO_PASID, 0);
	213	+ if (IS_ERR(group->attach_handle))
	214	+ goto err_abort;
	215	+ }
	216	+
	217	+ if (!group->attach_handle->domain->iopf_handler)
211	218	goto err_abort;
212	219
213	220	/*
214	221	* On success iopf_handler must call iopf_group_response() and
215	222	* iopf_free_group()
216	223	*/
217		− if (group->domain->iopf_handler(group))
	224	+ if (group->attach_handle->domain->iopf_handler(group))
218	225	goto err_abort;
219	226
220	227	return;
221	228
222	229	err_abort:
	230	+ dev_warn_ratelimited(dev, "iopf with pasid %d aborted\n",
	231	+ fault->prm.pasid);
223	232	iopf_group_response(group, IOMMU_PAGE_RESP_FAILURE);
224	233	if (group == &abort_group)
225	234	__iopf_free_group(group);

drivers/iommu/iommu-priv.h+11−0

@@ -28,4 +28,15 @@ void iommu_device_unregister_bus(struct iommu_device *iommu, · 27 unmodified lines
28	28	const struct bus_type *bus,
29	29	struct notifier_block *nb);
30	30
	31	+struct iommu_attach_handle iommu_attach_handle_get(struct iommu_group group,
	32	+ ioasid_t pasid,
	33	+ unsigned int type);
	34	+int iommu_attach_group_handle(struct iommu_domain *domain,
	35	+ struct iommu_group *group,
	36	+ struct iommu_attach_handle *handle);
	37	+void iommu_detach_group_handle(struct iommu_domain *domain,
	38	+ struct iommu_group *group);
	39	+int iommu_replace_group_handle(struct iommu_group *group,
	40	+ struct iommu_domain *new_domain,
	41	+ struct iommu_attach_handle *handle);
31	42	#endif /* __LINUX_IOMMU_PRIV_H */

drivers/iommu/iommu-sva.c+28−14

@@ -41,7 +41,6 @@ static struct iommu_mm_data iommu_alloc_mm_data(struct mm_struct mm, struct de · 40 unmodified lines
41	41	}
42	42	iommu_mm->pasid = pasid;
43	43	INIT_LIST_HEAD(&iommu_mm->sva_domains);
44		− INIT_LIST_HEAD(&iommu_mm->sva_handles);
45	44	/*
46	45	* Make sure the write to mm->iommu_mm is not reordered in front of
47	46	* initialization to iommu_mm fields. If it does, readers may see a
@@ -69,11 +68,16 @@ static struct iommu_mm_data iommu_alloc_mm_data(struct mm_struct mm, struct de · 21 unmodified lines
69	68	*/
70	69	struct iommu_sva iommu_sva_bind_device(struct device dev, struct mm_struct *mm)
71	70	{
	71	+ struct iommu_group *group = dev->iommu_group;
	72	+ struct iommu_attach_handle *attach_handle;
72	73	struct iommu_mm_data *iommu_mm;
73	74	struct iommu_domain *domain;
74	75	struct iommu_sva *handle;
75	76	int ret;
76	77
	78	+ if (!group)
	79	+ return ERR_PTR(-ENODEV);
	80	+
77	81	mutex_lock(&iommu_sva_lock);
78	82
79	83	/* Allocate mm->pasid if necessary. */
@@ -83,12 +87,22 @@ struct iommu_sva iommu_sva_bind_device(struct device dev, struct mm_struct *mm · 3 unmodified lines
83	87	goto out_unlock;
84	88	}
85	89
86		− list_for_each_entry(handle, &mm->iommu_mm->sva_handles, handle_item) {
87		− if (handle->dev == dev) {
88		− refcount_inc(&handle->users);
89		− mutex_unlock(&iommu_sva_lock);
90		− return handle;
	90	+ /* A bond already exists, just take a reference`. */
	91	+ attach_handle = iommu_attach_handle_get(group, iommu_mm->pasid, IOMMU_DOMAIN_SVA);
	92	+ if (!IS_ERR(attach_handle)) {
	93	+ handle = container_of(attach_handle, struct iommu_sva, handle);
	94	+ if (attach_handle->domain->mm != mm) {
	95	+ ret = -EBUSY;
	96	+ goto out_unlock;
91	97	}
	98	+ refcount_inc(&handle->users);
	99	+ mutex_unlock(&iommu_sva_lock);
	100	+ return handle;
	101	+ }
	102	+
	103	+ if (PTR_ERR(attach_handle) != -ENOENT) {
	104	+ ret = PTR_ERR(attach_handle);
	105	+ goto out_unlock;
92	106	}
93	107
94	108	handle = kzalloc(sizeof(*handle), GFP_KERNEL);
@@ -99,7 +113,8 @@ struct iommu_sva iommu_sva_bind_device(struct device dev, struct mm_struct *mm · 4 unmodified lines
99	113
100	114	/* Search for an existing domain. */
101	115	list_for_each_entry(domain, &mm->iommu_mm->sva_domains, next) {
102		− ret = iommu_attach_device_pasid(domain, dev, iommu_mm->pasid);
	116	+ ret = iommu_attach_device_pasid(domain, dev, iommu_mm->pasid,
	117	+ &handle->handle);
103	118	if (!ret) {
104	119	domain->users++;
105	120	goto out;
@@ -113,7 +128,8 @@ struct iommu_sva iommu_sva_bind_device(struct device dev, struct mm_struct *mm · 7 unmodified lines
113	128	goto out_free_handle;
114	129	}
115	130
116		− ret = iommu_attach_device_pasid(domain, dev, iommu_mm->pasid);
	131	+ ret = iommu_attach_device_pasid(domain, dev, iommu_mm->pasid,
	132	+ &handle->handle);
117	133	if (ret)
118	134	goto out_free_domain;
119	135	domain->users = 1;
@@ -121,10 +137,8 @@ struct iommu_sva iommu_sva_bind_device(struct device dev, struct mm_struct *mm · 1 unmodified line
121	137
122	138	out:
123	139	refcount_set(&handle->users, 1);
124		− list_add(&handle->handle_item, &mm->iommu_mm->sva_handles);
125	140	mutex_unlock(&iommu_sva_lock);
126	141	handle->dev = dev;
127		− handle->domain = domain;
128	142	return handle;
129	143
130	144	out_free_domain:
@@ -147,7 +161,7 @@ EXPORT_SYMBOL_GPL(iommu_sva_bind_device); · 16 unmodified lines
147	161	*/
148	162	void iommu_sva_unbind_device(struct iommu_sva *handle)
149	163	{
150		− struct iommu_domain *domain = handle->domain;
	164	+ struct iommu_domain *domain = handle->handle.domain;
151	165	struct iommu_mm_data *iommu_mm = domain->mm->iommu_mm;
152	166	struct device *dev = handle->dev;
153	167
@@ -156,7 +170,6 @@ void iommu_sva_unbind_device(struct iommu_sva *handle) · 2 unmodified lines
156	170	mutex_unlock(&iommu_sva_lock);
157	171	return;
158	172	}
159		− list_del(&handle->handle_item);
160	173
161	174	iommu_detach_device_pasid(domain, dev, iommu_mm->pasid);
162	175	if (--domain->users == 0) {
@@ -170,7 +183,7 @@ EXPORT_SYMBOL_GPL(iommu_sva_unbind_device); · 7 unmodified lines
170	183
171	184	u32 iommu_sva_get_pasid(struct iommu_sva *handle)
172	185	{
173		− struct iommu_domain *domain = handle->domain;
	186	+ struct iommu_domain *domain = handle->handle.domain;
174	187
175	188	return mm_get_enqcmd_pasid(domain->mm);
176	189	}
@@ -259,7 +272,8 @@ static void iommu_sva_handle_iopf(struct work_struct *work) · 82 unmodified lines
259	272	if (status != IOMMU_PAGE_RESP_SUCCESS)
260	273	break;
261	274
262		− status = iommu_sva_handle_mm(&iopf->fault, group->domain->mm);
	275	+ status = iommu_sva_handle_mm(&iopf->fault,
	276	+ group->attach_handle->domain->mm);
263	277	}
264	278
265	279	iopf_group_response(group, status);

drivers/iommu/iommu.c+143−42

@@ -3352,16 +3352,17 @@ static void __iommu_remove_group_pasid(struct iommu_group *group, · 3351 unmodified lines
3352	3352	* @domain: the iommu domain.
3353	3353	* @dev: the attached device.
3354	3354	* @pasid: the pasid of the device.
	3355	+ * @handle: the attach handle.
3355	3356	*
3356	3357	* Return: 0 on success, or an error.
3357	3358	*/
3358	3359	int iommu_attach_device_pasid(struct iommu_domain *domain,
3359		− struct device *dev, ioasid_t pasid)
	3360	+ struct device *dev, ioasid_t pasid,
	3361	+ struct iommu_attach_handle *handle)
3360	3362	{
3361	3363	/* Caller must be a probed driver on dev */
3362	3364	struct iommu_group *group = dev->iommu_group;
3363	3365	struct group_device *device;
3364		− void *curr;
3365	3366	int ret;
3366	3367
3367	3368	if (!domain->ops->set_dev_pasid)
@@ -3382,11 +3383,12 @@ int iommu_attach_device_pasid(struct iommu_domain *domain, · 14 unmodified lines
3382	3383	}
3383	3384	}
3384	3385
3385		− curr = xa_cmpxchg(&group->pasid_array, pasid, NULL, domain, GFP_KERNEL);
3386		− if (curr) {
3387		− ret = xa_err(curr) ? : -EBUSY;
	3386	+ if (handle)
	3387	+ handle->domain = domain;
	3388	+
	3389	+ ret = xa_insert(&group->pasid_array, pasid, handle, GFP_KERNEL);
	3390	+ if (ret)
3388	3391	goto out_unlock;
3389		− }
3390	3392
3391	3393	ret = __iommu_set_group_pasid(domain, group, pasid);
3392	3394	if (ret)
@@ -3414,46 +3416,11 @@ void iommu_detach_device_pasid(struct iommu_domain domain, struct device dev, · 21 unmodified lines
3414	3416
3415	3417	mutex_lock(&group->mutex);
3416	3418	__iommu_remove_group_pasid(group, pasid, domain);
3417		− WARN_ON(xa_erase(&group->pasid_array, pasid) != domain);
	3419	+ xa_erase(&group->pasid_array, pasid);
3418	3420	mutex_unlock(&group->mutex);
3419	3421	}
3420	3422	EXPORT_SYMBOL_GPL(iommu_detach_device_pasid);
3421	3423
3422		−/*
3423		− * iommu_get_domain_for_dev_pasid() - Retrieve domain for @pasid of @dev
3424		− * @dev: the queried device
3425		− * @pasid: the pasid of the device
3426		− * @type: matched domain type, 0 for any match
3427		− *
3428		− * This is a variant of iommu_get_domain_for_dev(). It returns the existing
3429		− * domain attached to pasid of a device. Callers must hold a lock around this
3430		− * function, and both iommu_attach/detach_dev_pasid() whenever a domain of
3431		− * type is being manipulated. This API does not internally resolve races with
3432		− * attach/detach.
3433		− *
3434		− * Return: attached domain on success, NULL otherwise.
3435		− */
3436		−struct iommu_domain iommu_get_domain_for_dev_pasid(struct device dev,
3437		− ioasid_t pasid,
3438		− unsigned int type)
3439		−{
3440		− /* Caller must be a probed driver on dev */
3441		− struct iommu_group *group = dev->iommu_group;
3442		− struct iommu_domain *domain;
3443		−
3444		− if (!group)
3445		− return NULL;
3446		−
3447		− xa_lock(&group->pasid_array);
3448		− domain = xa_load(&group->pasid_array, pasid);
3449		− if (type && domain && domain->type != type)
3450		− domain = ERR_PTR(-EBUSY);
3451		− xa_unlock(&group->pasid_array);
3452		−
3453		− return domain;
3454		−}
3455		−EXPORT_SYMBOL_GPL(iommu_get_domain_for_dev_pasid);
3456		−
3457	3424	ioasid_t iommu_alloc_global_pasid(struct device *dev)
3458	3425	{
3459	3426	int ret;
@@ -3480,3 +3447,137 @@ void iommu_free_global_pasid(ioasid_t pasid) · 20 unmodified lines
3480	3447	ida_free(&iommu_global_pasid_ida, pasid);
3481	3448	}
3482	3449	EXPORT_SYMBOL_GPL(iommu_free_global_pasid);
	3450	+
	3451	+/**
	3452	+ * iommu_attach_handle_get - Return the attach handle
	3453	+ * @group: the iommu group that domain was attached to
	3454	+ * @pasid: the pasid within the group
	3455	+ * @type: matched domain type, 0 for any match
	3456	+ *
	3457	+ * Return handle or ERR_PTR(-ENOENT) on none, ERR_PTR(-EBUSY) on mismatch.
	3458	+ *
	3459	+ * Return the attach handle to the caller. The life cycle of an iommu attach
	3460	+ * handle is from the time when the domain is attached to the time when the
	3461	+ * domain is detached. Callers are required to synchronize the call of
	3462	+ * iommu_attach_handle_get() with domain attachment and detachment. The attach
	3463	+ * handle can only be used during its life cycle.
	3464	+ */
	3465	+struct iommu_attach_handle *
	3466	+iommu_attach_handle_get(struct iommu_group *group, ioasid_t pasid, unsigned int type)
	3467	+{
	3468	+ struct iommu_attach_handle *handle;
	3469	+
	3470	+ xa_lock(&group->pasid_array);
	3471	+ handle = xa_load(&group->pasid_array, pasid);
	3472	+ if (!handle)
	3473	+ handle = ERR_PTR(-ENOENT);
	3474	+ else if (type && handle->domain->type != type)
	3475	+ handle = ERR_PTR(-EBUSY);
	3476	+ xa_unlock(&group->pasid_array);
	3477	+
	3478	+ return handle;
	3479	+}
	3480	+EXPORT_SYMBOL_NS_GPL(iommu_attach_handle_get, IOMMUFD_INTERNAL);
	3481	+
	3482	+/**
	3483	+ * iommu_attach_group_handle - Attach an IOMMU domain to an IOMMU group
	3484	+ * @domain: IOMMU domain to attach
	3485	+ * @group: IOMMU group that will be attached
	3486	+ * @handle: attach handle
	3487	+ *
	3488	+ * Returns 0 on success and error code on failure.
	3489	+ *
	3490	+ * This is a variant of iommu_attach_group(). It allows the caller to provide
	3491	+ * an attach handle and use it when the domain is attached. This is currently
	3492	+ * used by IOMMUFD to deliver the I/O page faults.
	3493	+ */
	3494	+int iommu_attach_group_handle(struct iommu_domain *domain,
	3495	+ struct iommu_group *group,
	3496	+ struct iommu_attach_handle *handle)
	3497	+{
	3498	+ int ret;
	3499	+
	3500	+ if (handle)
	3501	+ handle->domain = domain;
	3502	+
	3503	+ mutex_lock(&group->mutex);
	3504	+ ret = xa_insert(&group->pasid_array, IOMMU_NO_PASID, handle, GFP_KERNEL);
	3505	+ if (ret)
	3506	+ goto err_unlock;
	3507	+
	3508	+ ret = __iommu_attach_group(domain, group);
	3509	+ if (ret)
	3510	+ goto err_erase;
	3511	+ mutex_unlock(&group->mutex);
	3512	+
	3513	+ return 0;
	3514	+err_erase:
	3515	+ xa_erase(&group->pasid_array, IOMMU_NO_PASID);
	3516	+err_unlock:
	3517	+ mutex_unlock(&group->mutex);
	3518	+ return ret;
	3519	+}
	3520	+EXPORT_SYMBOL_NS_GPL(iommu_attach_group_handle, IOMMUFD_INTERNAL);
	3521	+
	3522	+/**
	3523	+ * iommu_detach_group_handle - Detach an IOMMU domain from an IOMMU group
	3524	+ * @domain: IOMMU domain to attach
	3525	+ * @group: IOMMU group that will be attached
	3526	+ *
	3527	+ * Detach the specified IOMMU domain from the specified IOMMU group.
	3528	+ * It must be used in conjunction with iommu_attach_group_handle().
	3529	+ */
	3530	+void iommu_detach_group_handle(struct iommu_domain *domain,
	3531	+ struct iommu_group *group)
	3532	+{
	3533	+ mutex_lock(&group->mutex);
	3534	+ __iommu_group_set_core_domain(group);
	3535	+ xa_erase(&group->pasid_array, IOMMU_NO_PASID);
	3536	+ mutex_unlock(&group->mutex);
	3537	+}
	3538	+EXPORT_SYMBOL_NS_GPL(iommu_detach_group_handle, IOMMUFD_INTERNAL);
	3539	+
	3540	+/**
	3541	+ * iommu_replace_group_handle - replace the domain that a group is attached to
	3542	+ * @group: IOMMU group that will be attached to the new domain
	3543	+ * @new_domain: new IOMMU domain to replace with
	3544	+ * @handle: attach handle
	3545	+ *
	3546	+ * This is a variant of iommu_group_replace_domain(). It allows the caller to
	3547	+ * provide an attach handle for the new domain and use it when the domain is
	3548	+ * attached.
	3549	+ */
	3550	+int iommu_replace_group_handle(struct iommu_group *group,
	3551	+ struct iommu_domain *new_domain,
	3552	+ struct iommu_attach_handle *handle)
	3553	+{
	3554	+ void *curr;
	3555	+ int ret;
	3556	+
	3557	+ if (!new_domain)
	3558	+ return -EINVAL;
	3559	+
	3560	+ mutex_lock(&group->mutex);
	3561	+ if (handle) {
	3562	+ ret = xa_reserve(&group->pasid_array, IOMMU_NO_PASID, GFP_KERNEL);
	3563	+ if (ret)
	3564	+ goto err_unlock;
	3565	+ }
	3566	+
	3567	+ ret = __iommu_group_set_domain(group, new_domain);
	3568	+ if (ret)
	3569	+ goto err_release;
	3570	+
	3571	+ curr = xa_store(&group->pasid_array, IOMMU_NO_PASID, handle, GFP_KERNEL);
	3572	+ WARN_ON(xa_is_err(curr));
	3573	+
	3574	+ mutex_unlock(&group->mutex);
	3575	+
	3576	+ return 0;
	3577	+err_release:
	3578	+ xa_release(&group->pasid_array, IOMMU_NO_PASID);
	3579	+err_unlock:
	3580	+ mutex_unlock(&group->mutex);
	3581	+ return ret;
	3582	+}
	3583	+EXPORT_SYMBOL_NS_GPL(iommu_replace_group_handle, IOMMUFD_INTERNAL);

drivers/iommu/iommufd/Makefile+1−0

@@ -1,6 +1,7 @@
1	1	# SPDX-License-Identifier: GPL-2.0-only
2	2	iommufd-y := \
3	3	device.o \
	4	+ fault.o \
4	5	hw_pagetable.o \
5	6	io_pagetable.o \
6	7	ioas.o \

drivers/iommu/iommufd/device.c+4−3

@@ -215,6 +215,7 @@ struct iommufd_device iommufd_device_bind(struct iommufd_ctx ictx, · 214 unmodified lines
215	215	refcount_inc(&idev->obj.users);
216	216	/* igroup refcount moves into iommufd_device */
217	217	idev->igroup = igroup;
	218	+ mutex_init(&idev->iopf_lock);
218	219
219	220	/*
220	221	* If the caller fails after this success it must call
@@ -376,7 +377,7 @@ int iommufd_hw_pagetable_attach(struct iommufd_hw_pagetable *hwpt, · 155 unmodified lines
376	377	* attachment.
377	378	*/
378	379	if (list_empty(&idev->igroup->device_list)) {
379		− rc = iommu_attach_group(hwpt->domain, idev->igroup->group);
	380	+ rc = iommufd_hwpt_attach_device(hwpt, idev);
380	381	if (rc)
381	382	goto err_unresv;
382	383	idev->igroup->hwpt = hwpt;
@@ -402,7 +403,7 @@ iommufd_hw_pagetable_detach(struct iommufd_device *idev) · 19 unmodified lines
402	403	mutex_lock(&idev->igroup->lock);
403	404	list_del(&idev->group_item);
404	405	if (list_empty(&idev->igroup->device_list)) {
405		− iommu_detach_group(hwpt->domain, idev->igroup->group);
	406	+ iommufd_hwpt_detach_device(hwpt, idev);
406	407	idev->igroup->hwpt = NULL;
407	408	}
408	409	if (hwpt_is_paging(hwpt))
@@ -497,7 +498,7 @@ iommufd_device_do_replace(struct iommufd_device *idev, · 88 unmodified lines
497	498	goto err_unlock;
498	499	}
499	500
500		− rc = iommu_group_replace_domain(igroup->group, hwpt->domain);
	501	+ rc = iommufd_hwpt_replace_device(idev, hwpt, old_hwpt);
501	502	if (rc)
502	503	goto err_unresv;
503	504

drivers/iommu/iommufd/fault.c+443−0

@@ -0,0 +1,443 @@
	1	+// SPDX-License-Identifier: GPL-2.0-only
	2	+/* Copyright (C) 2024 Intel Corporation
	3	+ */
	4	+#define pr_fmt(fmt) "iommufd: " fmt
	5	+
	6	+#include <linux/file.h>
	7	+#include <linux/fs.h>
	8	+#include <linux/module.h>
	9	+#include <linux/mutex.h>
	10	+#include <linux/iommufd.h>
	11	+#include <linux/pci.h>
	12	+#include <linux/poll.h>
	13	+#include <linux/anon_inodes.h>
	14	+#include <uapi/linux/iommufd.h>
	15	+
	16	+#include "../iommu-priv.h"
	17	+#include "iommufd_private.h"
	18	+
	19	+static int iommufd_fault_iopf_enable(struct iommufd_device *idev)
	20	+{
	21	+ struct device *dev = idev->dev;
	22	+ int ret;
	23	+
	24	+ /*
	25	+ * Once we turn on PCI/PRI support for VF, the response failure code
	26	+ * should not be forwarded to the hardware due to PRI being a shared
	27	+ * resource between PF and VFs. There is no coordination for this
	28	+ * shared capability. This waits for a vPRI reset to recover.
	29	+ */
	30	+ if (dev_is_pci(dev) && to_pci_dev(dev)->is_virtfn)
	31	+ return -EINVAL;
	32	+
	33	+ mutex_lock(&idev->iopf_lock);
	34	+ /* Device iopf has already been on. */
	35	+ if (++idev->iopf_enabled > 1) {
	36	+ mutex_unlock(&idev->iopf_lock);
	37	+ return 0;
	38	+ }
	39	+
	40	+ ret = iommu_dev_enable_feature(dev, IOMMU_DEV_FEAT_IOPF);
	41	+ if (ret)
	42	+ --idev->iopf_enabled;
	43	+ mutex_unlock(&idev->iopf_lock);
	44	+
	45	+ return ret;
	46	+}
	47	+
	48	+static void iommufd_fault_iopf_disable(struct iommufd_device *idev)
	49	+{
	50	+ mutex_lock(&idev->iopf_lock);
	51	+ if (!WARN_ON(idev->iopf_enabled == 0)) {
	52	+ if (--idev->iopf_enabled == 0)
	53	+ iommu_dev_disable_feature(idev->dev, IOMMU_DEV_FEAT_IOPF);
	54	+ }
	55	+ mutex_unlock(&idev->iopf_lock);
	56	+}
	57	+
	58	+static int __fault_domain_attach_dev(struct iommufd_hw_pagetable *hwpt,
	59	+ struct iommufd_device *idev)
	60	+{
	61	+ struct iommufd_attach_handle *handle;
	62	+ int ret;
	63	+
	64	+ handle = kzalloc(sizeof(*handle), GFP_KERNEL);
	65	+ if (!handle)
	66	+ return -ENOMEM;
	67	+
	68	+ handle->idev = idev;
	69	+ ret = iommu_attach_group_handle(hwpt->domain, idev->igroup->group,
	70	+ &handle->handle);
	71	+ if (ret)
	72	+ kfree(handle);
	73	+
	74	+ return ret;
	75	+}
	76	+
	77	+int iommufd_fault_domain_attach_dev(struct iommufd_hw_pagetable *hwpt,
	78	+ struct iommufd_device *idev)
	79	+{
	80	+ int ret;
	81	+
	82	+ if (!hwpt->fault)
	83	+ return -EINVAL;
	84	+
	85	+ ret = iommufd_fault_iopf_enable(idev);
	86	+ if (ret)
	87	+ return ret;
	88	+
	89	+ ret = __fault_domain_attach_dev(hwpt, idev);
	90	+ if (ret)
	91	+ iommufd_fault_iopf_disable(idev);
	92	+
	93	+ return ret;
	94	+}
	95	+
	96	+static void iommufd_auto_response_faults(struct iommufd_hw_pagetable *hwpt,
	97	+ struct iommufd_attach_handle *handle)
	98	+{
	99	+ struct iommufd_fault *fault = hwpt->fault;
	100	+ struct iopf_group group, next;
	101	+ unsigned long index;
	102	+
	103	+ if (!fault)
	104	+ return;
	105	+
	106	+ mutex_lock(&fault->mutex);
	107	+ list_for_each_entry_safe(group, next, &fault->deliver, node) {
	108	+ if (group->attach_handle != &handle->handle)
	109	+ continue;
	110	+ list_del(&group->node);
	111	+ iopf_group_response(group, IOMMU_PAGE_RESP_INVALID);
	112	+ iopf_free_group(group);
	113	+ }
	114	+
	115	+ xa_for_each(&fault->response, index, group) {
	116	+ if (group->attach_handle != &handle->handle)
	117	+ continue;
	118	+ xa_erase(&fault->response, index);
	119	+ iopf_group_response(group, IOMMU_PAGE_RESP_INVALID);
	120	+ iopf_free_group(group);
	121	+ }
	122	+ mutex_unlock(&fault->mutex);
	123	+}
	124	+
	125	+static struct iommufd_attach_handle *
	126	+iommufd_device_get_attach_handle(struct iommufd_device *idev)
	127	+{
	128	+ struct iommu_attach_handle *handle;
	129	+
	130	+ handle = iommu_attach_handle_get(idev->igroup->group, IOMMU_NO_PASID, 0);
	131	+ if (IS_ERR(handle))
	132	+ return NULL;
	133	+
	134	+ return to_iommufd_handle(handle);
	135	+}
	136	+
	137	+void iommufd_fault_domain_detach_dev(struct iommufd_hw_pagetable *hwpt,
	138	+ struct iommufd_device *idev)
	139	+{
	140	+ struct iommufd_attach_handle *handle;
	141	+
	142	+ handle = iommufd_device_get_attach_handle(idev);
	143	+ iommu_detach_group_handle(hwpt->domain, idev->igroup->group);
	144	+ iommufd_auto_response_faults(hwpt, handle);
	145	+ iommufd_fault_iopf_disable(idev);
	146	+ kfree(handle);
	147	+}
	148	+
	149	+static int __fault_domain_replace_dev(struct iommufd_device *idev,
	150	+ struct iommufd_hw_pagetable *hwpt,
	151	+ struct iommufd_hw_pagetable *old)
	152	+{
	153	+ struct iommufd_attach_handle handle, curr = NULL;
	154	+ int ret;
	155	+
	156	+ if (old->fault)
	157	+ curr = iommufd_device_get_attach_handle(idev);
	158	+
	159	+ if (hwpt->fault) {
	160	+ handle = kzalloc(sizeof(*handle), GFP_KERNEL);
	161	+ if (!handle)
	162	+ return -ENOMEM;
	163	+
	164	+ handle->handle.domain = hwpt->domain;
	165	+ handle->idev = idev;
	166	+ ret = iommu_replace_group_handle(idev->igroup->group,
	167	+ hwpt->domain, &handle->handle);
	168	+ } else {
	169	+ ret = iommu_replace_group_handle(idev->igroup->group,
	170	+ hwpt->domain, NULL);
	171	+ }
	172	+
	173	+ if (!ret && curr) {
	174	+ iommufd_auto_response_faults(old, curr);
	175	+ kfree(curr);
	176	+ }
	177	+
	178	+ return ret;
	179	+}
	180	+
	181	+int iommufd_fault_domain_replace_dev(struct iommufd_device *idev,
	182	+ struct iommufd_hw_pagetable *hwpt,
	183	+ struct iommufd_hw_pagetable *old)
	184	+{
	185	+ bool iopf_off = !hwpt->fault && old->fault;
	186	+ bool iopf_on = hwpt->fault && !old->fault;
	187	+ int ret;
	188	+
	189	+ if (iopf_on) {
	190	+ ret = iommufd_fault_iopf_enable(idev);
	191	+ if (ret)
	192	+ return ret;
	193	+ }
	194	+
	195	+ ret = __fault_domain_replace_dev(idev, hwpt, old);
	196	+ if (ret) {
	197	+ if (iopf_on)
	198	+ iommufd_fault_iopf_disable(idev);
	199	+ return ret;
	200	+ }
	201	+
	202	+ if (iopf_off)
	203	+ iommufd_fault_iopf_disable(idev);
	204	+
	205	+ return 0;
	206	+}
	207	+
	208	+void iommufd_fault_destroy(struct iommufd_object *obj)
	209	+{
	210	+ struct iommufd_fault *fault = container_of(obj, struct iommufd_fault, obj);
	211	+ struct iopf_group group, next;
	212	+
	213	+ /*
	214	+ * The iommufd object's reference count is zero at this point.
	215	+ * We can be confident that no other threads are currently
	216	+ * accessing this pointer. Therefore, acquiring the mutex here
	217	+ * is unnecessary.
	218	+ */
	219	+ list_for_each_entry_safe(group, next, &fault->deliver, node) {
	220	+ list_del(&group->node);
	221	+ iopf_group_response(group, IOMMU_PAGE_RESP_INVALID);
	222	+ iopf_free_group(group);
	223	+ }
	224	+}
	225	+
	226	+static void iommufd_compose_fault_message(struct iommu_fault *fault,
	227	+ struct iommu_hwpt_pgfault *hwpt_fault,
	228	+ struct iommufd_device *idev,
	229	+ u32 cookie)
	230	+{
	231	+ hwpt_fault->flags = fault->prm.flags;
	232	+ hwpt_fault->dev_id = idev->obj.id;
	233	+ hwpt_fault->pasid = fault->prm.pasid;
	234	+ hwpt_fault->grpid = fault->prm.grpid;
	235	+ hwpt_fault->perm = fault->prm.perm;
	236	+ hwpt_fault->addr = fault->prm.addr;
	237	+ hwpt_fault->length = 0;
	238	+ hwpt_fault->cookie = cookie;
	239	+}
	240	+
	241	+static ssize_t iommufd_fault_fops_read(struct file filep, char __user buf,
	242	+ size_t count, loff_t *ppos)
	243	+{
	244	+ size_t fault_size = sizeof(struct iommu_hwpt_pgfault);
	245	+ struct iommufd_fault *fault = filep->private_data;
	246	+ struct iommu_hwpt_pgfault data;
	247	+ struct iommufd_device *idev;
	248	+ struct iopf_group *group;
	249	+ struct iopf_fault *iopf;
	250	+ size_t done = 0;
	251	+ int rc = 0;
	252	+
	253	+ if (*ppos \|\| count % fault_size)
	254	+ return -ESPIPE;
	255	+
	256	+ mutex_lock(&fault->mutex);
	257	+ while (!list_empty(&fault->deliver) && count > done) {
	258	+ group = list_first_entry(&fault->deliver,
	259	+ struct iopf_group, node);
	260	+
	261	+ if (group->fault_count * fault_size > count - done)
	262	+ break;
	263	+
	264	+ rc = xa_alloc(&fault->response, &group->cookie, group,
	265	+ xa_limit_32b, GFP_KERNEL);
	266	+ if (rc)
	267	+ break;
	268	+
	269	+ idev = to_iommufd_handle(group->attach_handle)->idev;
	270	+ list_for_each_entry(iopf, &group->faults, list) {
	271	+ iommufd_compose_fault_message(&iopf->fault,
	272	+ &data, idev,
	273	+ group->cookie);
	274	+ if (copy_to_user(buf + done, &data, fault_size)) {
	275	+ xa_erase(&fault->response, group->cookie);
	276	+ rc = -EFAULT;
	277	+ break;
	278	+ }
	279	+ done += fault_size;
	280	+ }
	281	+
	282	+ list_del(&group->node);
	283	+ }
	284	+ mutex_unlock(&fault->mutex);
	285	+
	286	+ return done == 0 ? rc : done;
	287	+}
	288	+
	289	+static ssize_t iommufd_fault_fops_write(struct file filep, const char __user buf,
	290	+ size_t count, loff_t *ppos)
	291	+{
	292	+ size_t response_size = sizeof(struct iommu_hwpt_page_response);
	293	+ struct iommufd_fault *fault = filep->private_data;
	294	+ struct iommu_hwpt_page_response response;
	295	+ struct iopf_group *group;
	296	+ size_t done = 0;
	297	+ int rc = 0;
	298	+
	299	+ if (*ppos \|\| count % response_size)
	300	+ return -ESPIPE;
	301	+
	302	+ mutex_lock(&fault->mutex);
	303	+ while (count > done) {
	304	+ rc = copy_from_user(&response, buf + done, response_size);
	305	+ if (rc)
	306	+ break;
	307	+
	308	+ static_assert((int)IOMMUFD_PAGE_RESP_SUCCESS ==
	309	+ (int)IOMMU_PAGE_RESP_SUCCESS);
	310	+ static_assert((int)IOMMUFD_PAGE_RESP_INVALID ==
	311	+ (int)IOMMU_PAGE_RESP_INVALID);
	312	+ if (response.code != IOMMUFD_PAGE_RESP_SUCCESS &&
	313	+ response.code != IOMMUFD_PAGE_RESP_INVALID) {
	314	+ rc = -EINVAL;
	315	+ break;
	316	+ }
	317	+
	318	+ group = xa_erase(&fault->response, response.cookie);
	319	+ if (!group) {
	320	+ rc = -EINVAL;
	321	+ break;
	322	+ }
	323	+
	324	+ iopf_group_response(group, response.code);
	325	+ iopf_free_group(group);
	326	+ done += response_size;
	327	+ }
	328	+ mutex_unlock(&fault->mutex);
	329	+
	330	+ return done == 0 ? rc : done;
	331	+}
	332	+
	333	+static __poll_t iommufd_fault_fops_poll(struct file *filep,
	334	+ struct poll_table_struct *wait)
	335	+{
	336	+ struct iommufd_fault *fault = filep->private_data;
	337	+ __poll_t pollflags = EPOLLOUT;
	338	+
	339	+ poll_wait(filep, &fault->wait_queue, wait);
	340	+ mutex_lock(&fault->mutex);
	341	+ if (!list_empty(&fault->deliver))
	342	+ pollflags \|= EPOLLIN \| EPOLLRDNORM;
	343	+ mutex_unlock(&fault->mutex);
	344	+
	345	+ return pollflags;
	346	+}
	347	+
	348	+static int iommufd_fault_fops_release(struct inode inode, struct file filep)
	349	+{
	350	+ struct iommufd_fault *fault = filep->private_data;
	351	+
	352	+ refcount_dec(&fault->obj.users);
	353	+ iommufd_ctx_put(fault->ictx);
	354	+ return 0;
	355	+}
	356	+
	357	+static const struct file_operations iommufd_fault_fops = {
	358	+ .owner = THIS_MODULE,
	359	+ .open = nonseekable_open,
	360	+ .read = iommufd_fault_fops_read,
	361	+ .write = iommufd_fault_fops_write,
	362	+ .poll = iommufd_fault_fops_poll,
	363	+ .release = iommufd_fault_fops_release,
	364	+ .llseek = no_llseek,
	365	+};
	366	+
	367	+int iommufd_fault_alloc(struct iommufd_ucmd *ucmd)
	368	+{
	369	+ struct iommu_fault_alloc *cmd = ucmd->cmd;
	370	+ struct iommufd_fault *fault;
	371	+ struct file *filep;
	372	+ int fdno;
	373	+ int rc;
	374	+
	375	+ if (cmd->flags)
	376	+ return -EOPNOTSUPP;
	377	+
	378	+ fault = iommufd_object_alloc(ucmd->ictx, fault, IOMMUFD_OBJ_FAULT);
	379	+ if (IS_ERR(fault))
	380	+ return PTR_ERR(fault);
	381	+
	382	+ fault->ictx = ucmd->ictx;
	383	+ INIT_LIST_HEAD(&fault->deliver);
	384	+ xa_init_flags(&fault->response, XA_FLAGS_ALLOC1);
	385	+ mutex_init(&fault->mutex);
	386	+ init_waitqueue_head(&fault->wait_queue);
	387	+
	388	+ filep = anon_inode_getfile("[iommufd-pgfault]", &iommufd_fault_fops,
	389	+ fault, O_RDWR);
	390	+ if (IS_ERR(filep)) {
	391	+ rc = PTR_ERR(filep);
	392	+ goto out_abort;
	393	+ }
	394	+
	395	+ refcount_inc(&fault->obj.users);
	396	+ iommufd_ctx_get(fault->ictx);
	397	+ fault->filep = filep;
	398	+
	399	+ fdno = get_unused_fd_flags(O_CLOEXEC);
	400	+ if (fdno < 0) {
	401	+ rc = fdno;
	402	+ goto out_fput;
	403	+ }
	404	+
	405	+ cmd->out_fault_id = fault->obj.id;
	406	+ cmd->out_fault_fd = fdno;
	407	+
	408	+ rc = iommufd_ucmd_respond(ucmd, sizeof(*cmd));
	409	+ if (rc)
	410	+ goto out_put_fdno;
	411	+ iommufd_object_finalize(ucmd->ictx, &fault->obj);
	412	+
	413	+ fd_install(fdno, fault->filep);
	414	+
	415	+ return 0;
	416	+out_put_fdno:
	417	+ put_unused_fd(fdno);
	418	+out_fput:
	419	+ fput(filep);
	420	+ refcount_dec(&fault->obj.users);
	421	+ iommufd_ctx_put(fault->ictx);
	422	+out_abort:
	423	+ iommufd_object_abort_and_destroy(ucmd->ictx, &fault->obj);
	424	+
	425	+ return rc;
	426	+}
	427	+
	428	+int iommufd_fault_iopf_handler(struct iopf_group *group)
	429	+{
	430	+ struct iommufd_hw_pagetable *hwpt;
	431	+ struct iommufd_fault *fault;
	432	+
	433	+ hwpt = group->attach_handle->domain->fault_data;
	434	+ fault = hwpt->fault;
	435	+
	436	+ mutex_lock(&fault->mutex);
	437	+ list_add_tail(&group->node, &fault->deliver);
	438	+ mutex_unlock(&fault->mutex);
	439	+
	440	+ wake_up_interruptible(&fault->wait_queue);
	441	+
	442	+ return 0;
	443	+}

drivers/iommu/iommufd/hw_pagetable.c+32−9

@@ -8,6 +8,15 @@ · 7 unmodified lines
8	8	#include "../iommu-priv.h"
9	9	#include "iommufd_private.h"
10	10
	11	+static void __iommufd_hwpt_destroy(struct iommufd_hw_pagetable *hwpt)
	12	+{
	13	+ if (hwpt->domain)
	14	+ iommu_domain_free(hwpt->domain);
	15	+
	16	+ if (hwpt->fault)
	17	+ refcount_dec(&hwpt->fault->obj.users);
	18	+}
	19	+
11	20	void iommufd_hwpt_paging_destroy(struct iommufd_object *obj)
12	21	{
13	22	struct iommufd_hwpt_paging *hwpt_paging =
@@ -22,9 +31,7 @@ void iommufd_hwpt_paging_destroy(struct iommufd_object *obj) · 8 unmodified lines
22	31	hwpt_paging->common.domain);
23	32	}
24	33
25		− if (hwpt_paging->common.domain)
26		− iommu_domain_free(hwpt_paging->common.domain);
27		−
	34	+ __iommufd_hwpt_destroy(&hwpt_paging->common);
28	35	refcount_dec(&hwpt_paging->ioas->obj.users);
29	36	}
30	37
@@ -49,9 +56,7 @@ void iommufd_hwpt_nested_destroy(struct iommufd_object *obj) · 18 unmodified lines
49	56	struct iommufd_hwpt_nested *hwpt_nested =
50	57	container_of(obj, struct iommufd_hwpt_nested, common.obj);
51	58
52		− if (hwpt_nested->common.domain)
53		− iommu_domain_free(hwpt_nested->common.domain);
54		−
	59	+ __iommufd_hwpt_destroy(&hwpt_nested->common);
55	60	refcount_dec(&hwpt_nested->parent->common.obj.users);
56	61	}
57	62
@@ -213,7 +218,8 @@ iommufd_hwpt_nested_alloc(struct iommufd_ctx *ictx, · 155 unmodified lines
213	218	struct iommufd_hw_pagetable *hwpt;
214	219	int rc;
215	220
216		− if (flags \|\| !user_data->len \|\| !ops->domain_alloc_user)
	221	+ if ((flags & ~IOMMU_HWPT_FAULT_ID_VALID) \|\|
	222	+ !user_data->len \|\| !ops->domain_alloc_user)
217	223	return ERR_PTR(-EOPNOTSUPP);
218	224	if (parent->auto_domain \|\| !parent->nest_parent)
219	225	return ERR_PTR(-EINVAL);
@@ -227,7 +233,8 @@ iommufd_hwpt_nested_alloc(struct iommufd_ctx *ictx, · 7 unmodified lines
227	233	refcount_inc(&parent->common.obj.users);
228	234	hwpt_nested->parent = parent;
229	235
230		− hwpt->domain = ops->domain_alloc_user(idev->dev, flags,
	236	+ hwpt->domain = ops->domain_alloc_user(idev->dev,
	237	+ flags & ~IOMMU_HWPT_FAULT_ID_VALID,
231	238	parent->common.domain, user_data);
232	239	if (IS_ERR(hwpt->domain)) {
233	240	rc = PTR_ERR(hwpt->domain);
@@ -236,7 +243,8 @@ iommufd_hwpt_nested_alloc(struct iommufd_ctx *ictx, · 2 unmodified lines
236	243	}
237	244	hwpt->domain->owner = ops;
238	245
239		− if (WARN_ON_ONCE(hwpt->domain->type != IOMMU_DOMAIN_NESTED)) {
	246	+ if (WARN_ON_ONCE(hwpt->domain->type != IOMMU_DOMAIN_NESTED \|\|
	247	+ !hwpt->domain->ops->cache_invalidate_user)) {
240	248	rc = -EINVAL;
241	249	goto out_abort;
242	250	}
@@ -308,6 +316,21 @@ int iommufd_hwpt_alloc(struct iommufd_ucmd *ucmd) · 65 unmodified lines
308	316	goto out_put_pt;
309	317	}
310	318
	319	+ if (cmd->flags & IOMMU_HWPT_FAULT_ID_VALID) {
	320	+ struct iommufd_fault *fault;
	321	+
	322	+ fault = iommufd_get_fault(ucmd, cmd->fault_id);
	323	+ if (IS_ERR(fault)) {
	324	+ rc = PTR_ERR(fault);
	325	+ goto out_hwpt;
	326	+ }
	327	+ hwpt->fault = fault;
	328	+ hwpt->domain->iopf_handler = iommufd_fault_iopf_handler;
	329	+ hwpt->domain->fault_data = hwpt;
	330	+ refcount_inc(&fault->obj.users);
	331	+ iommufd_put_object(ucmd->ictx, &fault->obj);
	332	+ }
	333	+
311	334	cmd->out_hwpt_id = hwpt->obj.id;
312	335	rc = iommufd_ucmd_respond(ucmd, sizeof(*cmd));
313	336	if (rc)

drivers/iommu/iommufd/iommufd_private.h+80−0

@@ -11,6 +11,7 @@ · 10 unmodified lines
11	11	#include <linux/iommu.h>
12	12	#include <linux/iova_bitmap.h>
13	13	#include <uapi/linux/iommufd.h>
	14	+#include "../iommu-priv.h"
14	15
15	16	struct iommu_domain;
16	17	struct iommu_group;
@@ -128,6 +129,7 @@ enum iommufd_object_type { · 111 unmodified lines
128	129	IOMMUFD_OBJ_HWPT_NESTED,
129	130	IOMMUFD_OBJ_IOAS,
130	131	IOMMUFD_OBJ_ACCESS,
	132	+ IOMMUFD_OBJ_FAULT,
131	133	#ifdef CONFIG_IOMMUFD_TEST
132	134	IOMMUFD_OBJ_SELFTEST,
133	135	#endif
@@ -292,6 +294,7 @@ int iommufd_check_iova_range(struct io_pagetable *iopt, · 158 unmodified lines
292	294	struct iommufd_hw_pagetable {
293	295	struct iommufd_object obj;
294	296	struct iommu_domain *domain;
	297	+ struct iommufd_fault *fault;
295	298	};
296	299
297	300	struct iommufd_hwpt_paging {
@@ -395,6 +398,9 @@ struct iommufd_device { · 97 unmodified lines
395	398	/* always the physical device */
396	399	struct device *dev;
397	400	bool enforce_cache_coherency;
	401	+ /* protect iopf_enabled counter */
	402	+ struct mutex iopf_lock;
	403	+ unsigned int iopf_enabled;
398	404	};
399	405
400	406	static inline struct iommufd_device *
@@ -426,6 +432,80 @@ void iopt_remove_access(struct io_pagetable *iopt, · 25 unmodified lines
426	432	u32 iopt_access_list_id);
427	433	void iommufd_access_destroy_object(struct iommufd_object *obj);
428	434
	435	+/*
	436	+ * An iommufd_fault object represents an interface to deliver I/O page faults
	437	+ * to the user space. These objects are created/destroyed by the user space and
	438	+ * associated with hardware page table objects during page-table allocation.
	439	+ */
	440	+struct iommufd_fault {
	441	+ struct iommufd_object obj;
	442	+ struct iommufd_ctx *ictx;
	443	+ struct file *filep;
	444	+
	445	+ /* The lists of outstanding faults protected by below mutex. */
	446	+ struct mutex mutex;
	447	+ struct list_head deliver;
	448	+ struct xarray response;
	449	+
	450	+ struct wait_queue_head wait_queue;
	451	+};
	452	+
	453	+struct iommufd_attach_handle {
	454	+ struct iommu_attach_handle handle;
	455	+ struct iommufd_device *idev;
	456	+};
	457	+
	458	+/* Convert an iommu attach handle to iommufd handle. */
	459	+#define to_iommufd_handle(hdl) container_of(hdl, struct iommufd_attach_handle, handle)
	460	+
	461	+static inline struct iommufd_fault *
	462	+iommufd_get_fault(struct iommufd_ucmd *ucmd, u32 id)
	463	+{
	464	+ return container_of(iommufd_get_object(ucmd->ictx, id,
	465	+ IOMMUFD_OBJ_FAULT),
	466	+ struct iommufd_fault, obj);
	467	+}
	468	+
	469	+int iommufd_fault_alloc(struct iommufd_ucmd *ucmd);
	470	+void iommufd_fault_destroy(struct iommufd_object *obj);
	471	+int iommufd_fault_iopf_handler(struct iopf_group *group);
	472	+
	473	+int iommufd_fault_domain_attach_dev(struct iommufd_hw_pagetable *hwpt,
	474	+ struct iommufd_device *idev);
	475	+void iommufd_fault_domain_detach_dev(struct iommufd_hw_pagetable *hwpt,
	476	+ struct iommufd_device *idev);
	477	+int iommufd_fault_domain_replace_dev(struct iommufd_device *idev,
	478	+ struct iommufd_hw_pagetable *hwpt,
	479	+ struct iommufd_hw_pagetable *old);
	480	+
	481	+static inline int iommufd_hwpt_attach_device(struct iommufd_hw_pagetable *hwpt,
	482	+ struct iommufd_device *idev)
	483	+{
	484	+ if (hwpt->fault)
	485	+ return iommufd_fault_domain_attach_dev(hwpt, idev);
	486	+
	487	+ return iommu_attach_group(hwpt->domain, idev->igroup->group);
	488	+}
	489	+
	490	+static inline void iommufd_hwpt_detach_device(struct iommufd_hw_pagetable *hwpt,
	491	+ struct iommufd_device *idev)
	492	+{
	493	+ if (hwpt->fault)
	494	+ iommufd_fault_domain_detach_dev(hwpt, idev);
	495	+
	496	+ iommu_detach_group(hwpt->domain, idev->igroup->group);
	497	+}
	498	+
	499	+static inline int iommufd_hwpt_replace_device(struct iommufd_device *idev,
	500	+ struct iommufd_hw_pagetable *hwpt,
	501	+ struct iommufd_hw_pagetable *old)
	502	+{
	503	+ if (old->fault \|\| hwpt->fault)
	504	+ return iommufd_fault_domain_replace_dev(idev, hwpt, old);
	505	+
	506	+ return iommu_group_replace_domain(idev->igroup->group, hwpt->domain);
	507	+}
	508	+
429	509	#ifdef CONFIG_IOMMUFD_TEST
430	510	int iommufd_test(struct iommufd_ucmd *ucmd);
431	511	void iommufd_selftest_destroy(struct iommufd_object *obj);

drivers/iommu/iommufd/iommufd_test.h+8−0

@@ -22,6 +22,7 @@ enum { · 21 unmodified lines
22	22	IOMMU_TEST_OP_MOCK_DOMAIN_FLAGS,
23	23	IOMMU_TEST_OP_DIRTY,
24	24	IOMMU_TEST_OP_MD_CHECK_IOTLB,
	25	+ IOMMU_TEST_OP_TRIGGER_IOPF,
25	26	};
26	27
27	28	enum {
@@ -127,6 +128,13 @@ struct iommu_test_cmd { · 99 unmodified lines
127	128	__u32 id;
128	129	__u32 iotlb;
129	130	} check_iotlb;
	131	+ struct {
	132	+ __u32 dev_id;
	133	+ __u32 pasid;
	134	+ __u32 grpid;
	135	+ __u32 perm;
	136	+ __u64 addr;
	137	+ } trigger_iopf;
130	138	};
131	139	__u32 last;
132	140	};

drivers/iommu/iommufd/iova_bitmap.c+49−75

@@ -35,6 +35,9 @@ struct iova_bitmap_map { · 34 unmodified lines
35	35	/* base IOVA representing bit 0 of the first page */
36	36	unsigned long iova;
37	37
	38	+ /* mapped length */
	39	+ unsigned long length;
	40	+
38	41	/* page size order that each bit granules to */
39	42	unsigned long pgshift;
40	43
@@ -113,9 +116,6 @@ struct iova_bitmap { · 72 unmodified lines
113	116
114	117	/* length of the IOVA range for the whole bitmap */
115	118	size_t length;
116		−
117		− /* length of the IOVA range set ahead the pinned pages */
118		− unsigned long set_ahead_length;
119	119	};
120	120
121	121	/*
@@ -156,6 +156,8 @@ static unsigned long iova_bitmap_mapped_iova(struct iova_bitmap *bitmap) · 34 unmodified lines
156	156	return bitmap->iova + iova_bitmap_index_to_offset(bitmap, skip);
157	157	}
158	158
	159	+static unsigned long iova_bitmap_mapped_length(struct iova_bitmap *bitmap);
	160	+
159	161	/*
160	162	* Pins the bitmap user pages for the current range window.
161	163	* This is internal to IOVA bitmap and called when advancing the
@@ -206,6 +208,7 @@ static int iova_bitmap_get(struct iova_bitmap *bitmap) · 44 unmodified lines
206	208	* aligned.
207	209	*/
208	210	mapped->pgoff = offset_in_page(addr);
	211	+ mapped->length = iova_bitmap_mapped_length(bitmap);
209	212	return 0;
210	213	}
211	214
@@ -263,9 +266,6 @@ struct iova_bitmap *iova_bitmap_alloc(unsigned long iova, size_t length, · 51 unmodified lines
263	266	goto err;
264	267	}
265	268
266		− rc = iova_bitmap_get(bitmap);
267		− if (rc)
268		− goto err;
269	269	return bitmap;
270	270
271	271	err:
@@ -338,65 +338,34 @@ static unsigned long iova_bitmap_mapped_length(struct iova_bitmap *bitmap) · 66 unmodified lines
338	338	}
339	339
340	340	/*
341		− * Returns true if there's not more data to iterate.
	341	+ * Returns true if [@iova..@iova+@length-1] is part of the mapped IOVA range.
342	342	*/
343		−static bool iova_bitmap_done(struct iova_bitmap *bitmap)
	343	+static bool iova_bitmap_mapped_range(struct iova_bitmap_map *mapped,
	344	+ unsigned long iova, size_t length)
344	345	{
345		− return bitmap->mapped_base_index >= bitmap->mapped_total_index;
346		−}
347		−
348		−static int iova_bitmap_set_ahead(struct iova_bitmap *bitmap,
349		− size_t set_ahead_length)
350		−{
351		− int ret = 0;
352		−
353		− while (set_ahead_length > 0 && !iova_bitmap_done(bitmap)) {
354		− unsigned long length = iova_bitmap_mapped_length(bitmap);
355		− unsigned long iova = iova_bitmap_mapped_iova(bitmap);
356		−
357		− ret = iova_bitmap_get(bitmap);
358		− if (ret)
359		− break;
360		−
361		− length = min(length, set_ahead_length);
362		− iova_bitmap_set(bitmap, iova, length);
363		−
364		− set_ahead_length -= length;
365		− bitmap->mapped_base_index +=
366		− iova_bitmap_offset_to_index(bitmap, length - 1) + 1;
367		− iova_bitmap_put(bitmap);
368		− }
369		−
370		− bitmap->set_ahead_length = 0;
371		− return ret;
	346	+ return mapped->npages &&
	347	+ (iova >= mapped->iova &&
	348	+ (iova + length - 1) <= (mapped->iova + mapped->length - 1));
372	349	}
373	350
374	351	/*
375		− * Advances to the next range, releases the current pinned
	352	+ * Advances to a selected range, releases the current pinned
376	353	* pages and pins the next set of bitmap pages.
377	354	* Returns 0 on success or otherwise errno.
378	355	*/
379		−static int iova_bitmap_advance(struct iova_bitmap *bitmap)
	356	+static int iova_bitmap_advance_to(struct iova_bitmap *bitmap,
	357	+ unsigned long iova)
380	358	{
381		− unsigned long iova = iova_bitmap_mapped_length(bitmap) - 1;
382		− unsigned long count = iova_bitmap_offset_to_index(bitmap, iova) + 1;
	359	+ unsigned long index;
383	360
384		− bitmap->mapped_base_index += count;
	361	+ index = iova_bitmap_offset_to_index(bitmap, iova - bitmap->iova);
	362	+ if (index >= bitmap->mapped_total_index)
	363	+ return -EINVAL;
	364	+ bitmap->mapped_base_index = index;
385	365
386	366	iova_bitmap_put(bitmap);
387		− if (iova_bitmap_done(bitmap))
388		− return 0;
389		−
390		− /* Iterate, set and skip any bits requested for next iteration */
391		− if (bitmap->set_ahead_length) {
392		− int ret;
393		−
394		− ret = iova_bitmap_set_ahead(bitmap, bitmap->set_ahead_length);
395		− if (ret)
396		− return ret;
397		− }
398	367
399		− /* When advancing the index we pin the next set of bitmap pages */
	368	+ /* Pin the next set of bitmap pages */
400	369	return iova_bitmap_get(bitmap);
401	370	}
402	371
@@ -416,17 +385,7 @@ static int iova_bitmap_advance(struct iova_bitmap *bitmap) · 13 unmodified lines
416	385	int iova_bitmap_for_each(struct iova_bitmap bitmap, void opaque,
417	386	iova_bitmap_fn_t fn)
418	387	{
419		− int ret = 0;
420		−
421		− for (; !iova_bitmap_done(bitmap) && !ret;
422		− ret = iova_bitmap_advance(bitmap)) {
423		− ret = fn(bitmap, iova_bitmap_mapped_iova(bitmap),
424		− iova_bitmap_mapped_length(bitmap), opaque);
425		− if (ret)
426		− break;
427		− }
428		−
429		− return ret;
	388	+ return fn(bitmap, bitmap->iova, bitmap->length, opaque);
430	389	}
431	390	EXPORT_SYMBOL_NS_GPL(iova_bitmap_for_each, IOMMUFD);
432	391
@@ -444,11 +403,25 @@ void iova_bitmap_set(struct iova_bitmap *bitmap, · 11 unmodified lines
444	403	unsigned long iova, size_t length)
445	404	{
446	405	struct iova_bitmap_map *mapped = &bitmap->mapped;
447		− unsigned long cur_bit = ((iova - mapped->iova) >>
448		− mapped->pgshift) + mapped->pgoff * BITS_PER_BYTE;
449		− unsigned long last_bit = (((iova + length - 1) - mapped->iova) >>
450		− mapped->pgshift) + mapped->pgoff * BITS_PER_BYTE;
451		− unsigned long last_page_idx = mapped->npages - 1;
	406	+ unsigned long cur_bit, last_bit, last_page_idx;
	407	+
	408	+update_indexes:
	409	+ if (unlikely(!iova_bitmap_mapped_range(mapped, iova, length))) {
	410	+
	411	+ /*
	412	+ * The attempt to advance the base index to @iova
	413	+ * may fail if it's out of bounds, or pinning the pages
	414	+ * returns an error.
	415	+ */
	416	+ if (iova_bitmap_advance_to(bitmap, iova))
	417	+ return;
	418	+ }
	419	+
	420	+ last_page_idx = mapped->npages - 1;
	421	+ cur_bit = ((iova - mapped->iova) >>
	422	+ mapped->pgshift) + mapped->pgoff * BITS_PER_BYTE;
	423	+ last_bit = (((iova + length - 1) - mapped->iova) >>
	424	+ mapped->pgshift) + mapped->pgoff * BITS_PER_BYTE;
452	425
453	426	do {
454	427	unsigned int page_idx = cur_bit / BITS_PER_PAGE;
@@ -457,18 +430,19 @@ void iova_bitmap_set(struct iova_bitmap *bitmap, · 2 unmodified lines
457	430	last_bit - cur_bit + 1);
458	431	void *kaddr;
459	432
460		− if (unlikely(page_idx > last_page_idx))
461		− break;
	433	+ if (unlikely(page_idx > last_page_idx)) {
	434	+ unsigned long left =
	435	+ ((last_bit - cur_bit + 1) << mapped->pgshift);
	436	+
	437	+ iova += (length - left);
	438	+ length = left;
	439	+ goto update_indexes;
	440	+ }
462	441
463	442	kaddr = kmap_local_page(mapped->pages[page_idx]);
464	443	bitmap_set(kaddr, offset, nbits);
465	444	kunmap_local(kaddr);
466	445	cur_bit += nbits;
467	446	} while (cur_bit <= last_bit);
468		−
469		− if (unlikely(cur_bit <= last_bit)) {
470		− bitmap->set_ahead_length =
471		− ((last_bit - cur_bit + 1) << bitmap->mapped.pgshift);
472		− }
473	447	}
474	448	EXPORT_SYMBOL_NS_GPL(iova_bitmap_set, IOMMUFD);

drivers/iommu/iommufd/main.c+6−0

@@ -319,6 +319,7 @@ static int iommufd_option(struct iommufd_ucmd *ucmd) · 318 unmodified lines
319	319
320	320	union ucmd_buffer {
321	321	struct iommu_destroy destroy;
	322	+ struct iommu_fault_alloc fault;
322	323	struct iommu_hw_info info;
323	324	struct iommu_hwpt_alloc hwpt;
324	325	struct iommu_hwpt_get_dirty_bitmap get_dirty_bitmap;
@@ -355,6 +356,8 @@ struct iommufd_ioctl_op { · 30 unmodified lines
355	356	}
356	357	static const struct iommufd_ioctl_op iommufd_ioctl_ops[] = {
357	358	IOCTL_OP(IOMMU_DESTROY, iommufd_destroy, struct iommu_destroy, id),
	359	+ IOCTL_OP(IOMMU_FAULT_QUEUE_ALLOC, iommufd_fault_alloc, struct iommu_fault_alloc,
	360	+ out_fault_fd),
358	361	IOCTL_OP(IOMMU_GET_HW_INFO, iommufd_get_hw_info, struct iommu_hw_info,
359	362	__reserved),
360	363	IOCTL_OP(IOMMU_HWPT_ALLOC, iommufd_hwpt_alloc, struct iommu_hwpt_alloc,
@@ -513,6 +516,9 @@ static const struct iommufd_object_ops iommufd_object_ops[] = { · 152 unmodified lines
513	516	.destroy = iommufd_hwpt_nested_destroy,
514	517	.abort = iommufd_hwpt_nested_abort,
515	518	},
	519	+ [IOMMUFD_OBJ_FAULT] = {
	520	+ .destroy = iommufd_fault_destroy,
	521	+ },
516	522	#ifdef CONFIG_IOMMUFD_TEST
517	523	[IOMMUFD_OBJ_SELFTEST] = {
518	524	.destroy = iommufd_selftest_destroy,

drivers/iommu/iommufd/selftest.c+67−3

@@ -266,8 +266,8 @@ static int mock_domain_read_and_clear_dirty(struct iommu_domain *domain, · 265 unmodified lines
266	266
267	267	/* Clear dirty */
268	268	if (mock_test_and_clear_dirty(mock, head, pgsize, flags))
269		− iommu_dirty_bitmap_record(dirty, head, pgsize);
270		− iova = head + pgsize;
	269	+ iommu_dirty_bitmap_record(dirty, iova, pgsize);
	270	+ iova += pgsize;
271	271	} while (iova < end);
272	272
273	273	return 0;
@@ -504,6 +504,8 @@ static bool mock_domain_capable(struct device *dev, enum iommu_cap cap) · 230 unmodified lines
504	504	return false;
505	505	}
506	506
	507	+static struct iopf_queue *mock_iommu_iopf_queue;
	508	+
507	509	static struct iommu_device mock_iommu_device = {
508	510	};
509	511
@@ -514,6 +516,29 @@ static struct iommu_device mock_probe_device(struct device dev) · 4 unmodified lines
514	516	return &mock_iommu_device;
515	517	}
516	518
	519	+static void mock_domain_page_response(struct device dev, struct iopf_fault evt,
	520	+ struct iommu_page_response *msg)
	521	+{
	522	+}
	523	+
	524	+static int mock_dev_enable_feat(struct device *dev, enum iommu_dev_features feat)
	525	+{
	526	+ if (feat != IOMMU_DEV_FEAT_IOPF \|\| !mock_iommu_iopf_queue)
	527	+ return -ENODEV;
	528	+
	529	+ return iopf_queue_add_device(mock_iommu_iopf_queue, dev);
	530	+}
	531	+
	532	+static int mock_dev_disable_feat(struct device *dev, enum iommu_dev_features feat)
	533	+{
	534	+ if (feat != IOMMU_DEV_FEAT_IOPF \|\| !mock_iommu_iopf_queue)
	535	+ return -ENODEV;
	536	+
	537	+ iopf_queue_remove_device(mock_iommu_iopf_queue, dev);
	538	+
	539	+ return 0;
	540	+}
	541	+
517	542	static const struct iommu_ops mock_ops = {
518	543	/*
519	544	* IOMMU_DOMAIN_BLOCKED cannot be returned from def_domain_type()
@@ -529,6 +554,10 @@ static const struct iommu_ops mock_ops = { · 9 unmodified lines
529	554	.capable = mock_domain_capable,
530	555	.device_group = generic_device_group,
531	556	.probe_device = mock_probe_device,
	557	+ .page_response = mock_domain_page_response,
	558	+ .dev_enable_feat = mock_dev_enable_feat,
	559	+ .dev_disable_feat = mock_dev_disable_feat,
	560	+ .user_pasid_table = true,
532	561	.default_domain_ops =
533	562	&(struct iommu_domain_ops){
534	563	.free = mock_domain_free,
@@ -1334,7 +1363,7 @@ static int iommufd_test_dirty(struct iommufd_ucmd *ucmd, unsigned int mockpt_id, · 799 unmodified lines
1334	1363	}
1335	1364
1336	1365	max = length / page_size;
1337		− bitmap_size = max / BITS_PER_BYTE;
	1366	+ bitmap_size = DIV_ROUND_UP(max, BITS_PER_BYTE);
1338	1367
1339	1368	tmp = kvzalloc(bitmap_size, GFP_KERNEL_ACCOUNT);
1340	1369	if (!tmp) {
@@ -1375,6 +1404,31 @@ out_put: · 34 unmodified lines
1375	1404	return rc;
1376	1405	}
1377	1406
	1407	+static int iommufd_test_trigger_iopf(struct iommufd_ucmd *ucmd,
	1408	+ struct iommu_test_cmd *cmd)
	1409	+{
	1410	+ struct iopf_fault event = { };
	1411	+ struct iommufd_device *idev;
	1412	+
	1413	+ idev = iommufd_get_device(ucmd, cmd->trigger_iopf.dev_id);
	1414	+ if (IS_ERR(idev))
	1415	+ return PTR_ERR(idev);
	1416	+
	1417	+ event.fault.prm.flags = IOMMU_FAULT_PAGE_REQUEST_LAST_PAGE;
	1418	+ if (cmd->trigger_iopf.pasid != IOMMU_NO_PASID)
	1419	+ event.fault.prm.flags \|= IOMMU_FAULT_PAGE_REQUEST_PASID_VALID;
	1420	+ event.fault.type = IOMMU_FAULT_PAGE_REQ;
	1421	+ event.fault.prm.addr = cmd->trigger_iopf.addr;
	1422	+ event.fault.prm.pasid = cmd->trigger_iopf.pasid;
	1423	+ event.fault.prm.grpid = cmd->trigger_iopf.grpid;
	1424	+ event.fault.prm.perm = cmd->trigger_iopf.perm;
	1425	+
	1426	+ iommu_report_device_fault(idev->dev, &event);
	1427	+ iommufd_put_object(ucmd->ictx, &idev->obj);
	1428	+
	1429	+ return 0;
	1430	+}
	1431	+
1378	1432	void iommufd_selftest_destroy(struct iommufd_object *obj)
1379	1433	{
1380	1434	struct selftest_obj *sobj = container_of(obj, struct selftest_obj, obj);
@@ -1450,6 +1504,8 @@ int iommufd_test(struct iommufd_ucmd *ucmd) · 69 unmodified lines
1450	1504	cmd->dirty.page_size,
1451	1505	u64_to_user_ptr(cmd->dirty.uptr),
1452	1506	cmd->dirty.flags);
	1507	+ case IOMMU_TEST_OP_TRIGGER_IOPF:
	1508	+ return iommufd_test_trigger_iopf(ucmd, cmd);
1453	1509	default:
1454	1510	return -EOPNOTSUPP;
1455	1511	}
@@ -1491,6 +1547,9 @@ int __init iommufd_test_init(void) · 35 unmodified lines
1491	1547	&iommufd_mock_bus_type.nb);
1492	1548	if (rc)
1493	1549	goto err_sysfs;
	1550	+
	1551	+ mock_iommu_iopf_queue = iopf_queue_alloc("mock-iopfq");
	1552	+
1494	1553	return 0;
1495	1554
1496	1555	err_sysfs:
@@ -1506,6 +1565,11 @@ err_dbgfs: · 9 unmodified lines
1506	1565
1507	1566	void iommufd_test_exit(void)
1508	1567	{
	1568	+ if (mock_iommu_iopf_queue) {
	1569	+ iopf_queue_free(mock_iommu_iopf_queue);
	1570	+ mock_iommu_iopf_queue = NULL;
	1571	+ }
	1572	+
1509	1573	iommu_device_sysfs_remove(&mock_iommu_device);
1510	1574	iommu_device_unregister_bus(&mock_iommu_device,
1511	1575	&iommufd_mock_bus_type.bus,

include/linux/iommu.h+25−16

@@ -124,12 +124,16 @@ struct iopf_fault { · 123 unmodified lines
124	124	struct iopf_group {
125	125	struct iopf_fault last_fault;
126	126	struct list_head faults;
	127	+ size_t fault_count;
127	128	/* list node for iommu_fault_param::faults */
128	129	struct list_head pending_node;
129	130	struct work_struct work;
130		− struct iommu_domain *domain;
	131	+ struct iommu_attach_handle *attach_handle;
131	132	/* The device's fault data parameter. */
132	133	struct iommu_fault_param *fault_param;
	134	+ /* Used by handler provider to hook the group on its own lists. */
	135	+ struct list_head node;
	136	+ u32 cookie;
133	137	};
134	138
135	139	/**
@@ -547,6 +551,10 @@ static inline int __iommu_copy_struct_from_user_array( · 411 unmodified lines
547	551	* @default_domain: If not NULL this will always be set as the default domain.
548	552	* This should be an IDENTITY/BLOCKED/PLATFORM domain.
549	553	* Do not use in new drivers.
	554	+ * @user_pasid_table: IOMMU driver supports user-managed PASID table. There is
	555	+ * no user domain for each PASID and the I/O page faults are
	556	+ * forwarded through the user domain attached to the device
	557	+ * RID.
550	558	*/
551	559	struct iommu_ops {
552	560	bool (capable)(struct device dev, enum iommu_cap);
@@ -590,6 +598,7 @@ struct iommu_ops { · 37 unmodified lines
590	598	struct iommu_domain *blocked_domain;
591	599	struct iommu_domain *release_domain;
592	600	struct iommu_domain *default_domain;
	601	+ u8 user_pasid_table:1;
593	602	};
594	603
595	604	/**
@@ -989,20 +998,28 @@ struct iommu_fwspec { · 393 unmodified lines
989	998	/* ATS is supported */
990	999	#define IOMMU_FWSPEC_PCI_RC_ATS (1 << 0)
991	1000
	1001	+/*
	1002	+ * An iommu attach handle represents a relationship between an iommu domain
	1003	+ * and a PASID or RID of a device. It is allocated and managed by the component
	1004	+ * that manages the domain and is stored in the iommu group during the time the
	1005	+ * domain is attached.
	1006	+ */
	1007	+struct iommu_attach_handle {
	1008	+ struct iommu_domain *domain;
	1009	+};
	1010	+
992	1011	/**
993	1012	* struct iommu_sva - handle to a device-mm bond
994	1013	*/
995	1014	struct iommu_sva {
	1015	+ struct iommu_attach_handle handle;
996	1016	struct device *dev;
997		− struct iommu_domain *domain;
998		− struct list_head handle_item;
999	1017	refcount_t users;
1000	1018	};
1001	1019
1002	1020	struct iommu_mm_data {
1003	1021	u32 pasid;
1004	1022	struct list_head sva_domains;
1005		− struct list_head sva_handles;
1006	1023	};
1007	1024
1008	1025	int iommu_fwspec_init(struct device dev, struct fwnode_handle iommu_fwnode,
@@ -1052,12 +1069,10 @@ int iommu_device_claim_dma_owner(struct device dev, void owner); · 43 unmodified lines
1052	1069	void iommu_device_release_dma_owner(struct device *dev);
1053	1070
1054	1071	int iommu_attach_device_pasid(struct iommu_domain *domain,
1055		− struct device *dev, ioasid_t pasid);
	1072	+ struct device *dev, ioasid_t pasid,
	1073	+ struct iommu_attach_handle *handle);
1056	1074	void iommu_detach_device_pasid(struct iommu_domain *domain,
1057	1075	struct device *dev, ioasid_t pasid);
1058		−struct iommu_domain *
1059		−iommu_get_domain_for_dev_pasid(struct device *dev, ioasid_t pasid,
1060		− unsigned int type);
1061	1076	ioasid_t iommu_alloc_global_pasid(struct device *dev);
1062	1077	void iommu_free_global_pasid(ioasid_t pasid);
1063	1078	#else /* CONFIG_IOMMU_API */
@@ -1388,7 +1403,8 @@ static inline int iommu_device_claim_dma_owner(struct device dev, void owner) · 324 unmodified lines
1388	1403	}
1389	1404
1390	1405	static inline int iommu_attach_device_pasid(struct iommu_domain *domain,
1391		− struct device *dev, ioasid_t pasid)
	1406	+ struct device *dev, ioasid_t pasid,
	1407	+ struct iommu_attach_handle *handle)
1392	1408	{
1393	1409	return -ENODEV;
1394	1410	}
@@ -1398,13 +1414,6 @@ static inline void iommu_detach_device_pasid(struct iommu_domain *domain, · 3 unmodified lines
1398	1414	{
1399	1415	}
1400	1416
1401		−static inline struct iommu_domain *
1402		−iommu_get_domain_for_dev_pasid(struct device *dev, ioasid_t pasid,
1403		− unsigned int type)
1404		−{
1405		− return NULL;
1406		−}
1407		−
1408	1417	static inline ioasid_t iommu_alloc_global_pasid(struct device *dev)
1409	1418	{
1410	1419	return IOMMU_PASID_INVALID;

include/uapi/linux/iommufd.h+123−18

@@ -37,19 +37,20 @@ · 36 unmodified lines
37	37	enum {
38	38	IOMMUFD_CMD_BASE = 0x80,
39	39	IOMMUFD_CMD_DESTROY = IOMMUFD_CMD_BASE,
40		− IOMMUFD_CMD_IOAS_ALLOC,
41		− IOMMUFD_CMD_IOAS_ALLOW_IOVAS,
42		− IOMMUFD_CMD_IOAS_COPY,
43		− IOMMUFD_CMD_IOAS_IOVA_RANGES,
44		− IOMMUFD_CMD_IOAS_MAP,
45		− IOMMUFD_CMD_IOAS_UNMAP,
46		− IOMMUFD_CMD_OPTION,
47		− IOMMUFD_CMD_VFIO_IOAS,
48		− IOMMUFD_CMD_HWPT_ALLOC,
49		− IOMMUFD_CMD_GET_HW_INFO,
50		− IOMMUFD_CMD_HWPT_SET_DIRTY_TRACKING,
51		− IOMMUFD_CMD_HWPT_GET_DIRTY_BITMAP,
52		− IOMMUFD_CMD_HWPT_INVALIDATE,
	40	+ IOMMUFD_CMD_IOAS_ALLOC = 0x81,
	41	+ IOMMUFD_CMD_IOAS_ALLOW_IOVAS = 0x82,
	42	+ IOMMUFD_CMD_IOAS_COPY = 0x83,
	43	+ IOMMUFD_CMD_IOAS_IOVA_RANGES = 0x84,
	44	+ IOMMUFD_CMD_IOAS_MAP = 0x85,
	45	+ IOMMUFD_CMD_IOAS_UNMAP = 0x86,
	46	+ IOMMUFD_CMD_OPTION = 0x87,
	47	+ IOMMUFD_CMD_VFIO_IOAS = 0x88,
	48	+ IOMMUFD_CMD_HWPT_ALLOC = 0x89,
	49	+ IOMMUFD_CMD_GET_HW_INFO = 0x8a,
	50	+ IOMMUFD_CMD_HWPT_SET_DIRTY_TRACKING = 0x8b,
	51	+ IOMMUFD_CMD_HWPT_GET_DIRTY_BITMAP = 0x8c,
	52	+ IOMMUFD_CMD_HWPT_INVALIDATE = 0x8d,
	53	+ IOMMUFD_CMD_FAULT_QUEUE_ALLOC = 0x8e,
53	54	};
54	55
55	56	/**
@@ -356,10 +357,13 @@ struct iommu_vfio_ioas { · 300 unmodified lines
356	357	* the parent HWPT in a nesting configuration.
357	358	* @IOMMU_HWPT_ALLOC_DIRTY_TRACKING: Dirty tracking support for device IOMMU is
358	359	* enforced on device attachment
	360	+ * @IOMMU_HWPT_FAULT_ID_VALID: The fault_id field of hwpt allocation data is
	361	+ * valid.
359	362	*/
360	363	enum iommufd_hwpt_alloc_flags {
361	364	IOMMU_HWPT_ALLOC_NEST_PARENT = 1 << 0,
362	365	IOMMU_HWPT_ALLOC_DIRTY_TRACKING = 1 << 1,
	366	+ IOMMU_HWPT_FAULT_ID_VALID = 1 << 2,
363	367	};
364	368
365	369	/**
@@ -396,8 +400,8 @@ struct iommu_hwpt_vtd_s1 { · 30 unmodified lines
396	400	* @IOMMU_HWPT_DATA_VTD_S1: Intel VT-d stage-1 page table
397	401	*/
398	402	enum iommu_hwpt_data_type {
399		− IOMMU_HWPT_DATA_NONE,
400		− IOMMU_HWPT_DATA_VTD_S1,
	403	+ IOMMU_HWPT_DATA_NONE = 0,
	404	+ IOMMU_HWPT_DATA_VTD_S1 = 1,
401	405	};
402	406
403	407	/**
@@ -411,6 +415,9 @@ enum iommu_hwpt_data_type { · 7 unmodified lines
411	415	* @data_type: One of enum iommu_hwpt_data_type
412	416	* @data_len: Length of the type specific data
413	417	* @data_uptr: User pointer to the type specific data
	418	+ * @fault_id: The ID of IOMMUFD_FAULT object. Valid only if flags field of
	419	+ * IOMMU_HWPT_FAULT_ID_VALID is set.
	420	+ * @__reserved2: Padding to 64-bit alignment. Must be 0.
414	421	*
415	422	* Explicitly allocate a hardware page table object. This is the same object
416	423	* type that is returned by iommufd_device_attach() and represents the
@@ -441,6 +448,8 @@ struct iommu_hwpt_alloc { · 24 unmodified lines
441	448	__u32 data_type;
442	449	__u32 data_len;
443	450	__aligned_u64 data_uptr;
	451	+ __u32 fault_id;
	452	+ __u32 __reserved2;
444	453	};
445	454	#define IOMMU_HWPT_ALLOC _IO(IOMMUFD_TYPE, IOMMUFD_CMD_HWPT_ALLOC)
446	455
@@ -482,8 +491,8 @@ struct iommu_hw_info_vtd { · 35 unmodified lines
482	491	* @IOMMU_HW_INFO_TYPE_INTEL_VTD: Intel VT-d iommu info type
483	492	*/
484	493	enum iommu_hw_info_type {
485		− IOMMU_HW_INFO_TYPE_NONE,
486		− IOMMU_HW_INFO_TYPE_INTEL_VTD,
	494	+ IOMMU_HW_INFO_TYPE_NONE = 0,
	495	+ IOMMU_HW_INFO_TYPE_INTEL_VTD = 1,
487	496	};
488	497
489	498	/**
@@ -620,7 +629,7 @@ struct iommu_hwpt_get_dirty_bitmap { · 130 unmodified lines
620	629	* @IOMMU_HWPT_INVALIDATE_DATA_VTD_S1: Invalidation data for VTD_S1
621	630	*/
622	631	enum iommu_hwpt_invalidate_data_type {
623		− IOMMU_HWPT_INVALIDATE_DATA_VTD_S1,
	632	+ IOMMU_HWPT_INVALIDATE_DATA_VTD_S1 = 0,
624	633	};
625	634
626	635	/**
@@ -692,4 +701,100 @@ struct iommu_hwpt_invalidate { · 65 unmodified lines
692	701	__u32 __reserved;
693	702	};
694	703	#define IOMMU_HWPT_INVALIDATE _IO(IOMMUFD_TYPE, IOMMUFD_CMD_HWPT_INVALIDATE)
	704	+
	705	+/**
	706	+ * enum iommu_hwpt_pgfault_flags - flags for struct iommu_hwpt_pgfault
	707	+ * @IOMMU_PGFAULT_FLAGS_PASID_VALID: The pasid field of the fault data is
	708	+ * valid.
	709	+ * @IOMMU_PGFAULT_FLAGS_LAST_PAGE: It's the last fault of a fault group.
	710	+ */
	711	+enum iommu_hwpt_pgfault_flags {
	712	+ IOMMU_PGFAULT_FLAGS_PASID_VALID = (1 << 0),
	713	+ IOMMU_PGFAULT_FLAGS_LAST_PAGE = (1 << 1),
	714	+};
	715	+
	716	+/**
	717	+ * enum iommu_hwpt_pgfault_perm - perm bits for struct iommu_hwpt_pgfault
	718	+ * @IOMMU_PGFAULT_PERM_READ: request for read permission
	719	+ * @IOMMU_PGFAULT_PERM_WRITE: request for write permission
	720	+ * @IOMMU_PGFAULT_PERM_EXEC: (PCIE 10.4.1) request with a PASID that has the
	721	+ * Execute Requested bit set in PASID TLP Prefix.
	722	+ * @IOMMU_PGFAULT_PERM_PRIV: (PCIE 10.4.1) request with a PASID that has the
	723	+ * Privileged Mode Requested bit set in PASID TLP
	724	+ * Prefix.
	725	+ */
	726	+enum iommu_hwpt_pgfault_perm {
	727	+ IOMMU_PGFAULT_PERM_READ = (1 << 0),
	728	+ IOMMU_PGFAULT_PERM_WRITE = (1 << 1),
	729	+ IOMMU_PGFAULT_PERM_EXEC = (1 << 2),
	730	+ IOMMU_PGFAULT_PERM_PRIV = (1 << 3),
	731	+};
	732	+
	733	+/**
	734	+ * struct iommu_hwpt_pgfault - iommu page fault data
	735	+ * @flags: Combination of enum iommu_hwpt_pgfault_flags
	736	+ * @dev_id: id of the originated device
	737	+ * @pasid: Process Address Space ID
	738	+ * @grpid: Page Request Group Index
	739	+ * @perm: Combination of enum iommu_hwpt_pgfault_perm
	740	+ * @addr: Fault address
	741	+ * @length: a hint of how much data the requestor is expecting to fetch. For
	742	+ * example, if the PRI initiator knows it is going to do a 10MB
	743	+ * transfer, it could fill in 10MB and the OS could pre-fault in
	744	+ * 10MB of IOVA. It's default to 0 if there's no such hint.
	745	+ * @cookie: kernel-managed cookie identifying a group of fault messages. The
	746	+ * cookie number encoded in the last page fault of the group should
	747	+ * be echoed back in the response message.
	748	+ */
	749	+struct iommu_hwpt_pgfault {
	750	+ __u32 flags;
	751	+ __u32 dev_id;
	752	+ __u32 pasid;
	753	+ __u32 grpid;
	754	+ __u32 perm;
	755	+ __u64 addr;
	756	+ __u32 length;
	757	+ __u32 cookie;
	758	+};
	759	+
	760	+/**
	761	+ * enum iommufd_page_response_code - Return status of fault handlers
	762	+ * @IOMMUFD_PAGE_RESP_SUCCESS: Fault has been handled and the page tables
	763	+ * populated, retry the access. This is the
	764	+ * "Success" defined in PCI 10.4.2.1.
	765	+ * @IOMMUFD_PAGE_RESP_INVALID: Could not handle this fault, don't retry the
	766	+ * access. This is the "Invalid Request" in PCI
	767	+ * 10.4.2.1.
	768	+ */
	769	+enum iommufd_page_response_code {
	770	+ IOMMUFD_PAGE_RESP_SUCCESS = 0,
	771	+ IOMMUFD_PAGE_RESP_INVALID = 1,
	772	+};
	773	+
	774	+/**
	775	+ * struct iommu_hwpt_page_response - IOMMU page fault response
	776	+ * @cookie: The kernel-managed cookie reported in the fault message.
	777	+ * @code: One of response code in enum iommufd_page_response_code.
	778	+ */
	779	+struct iommu_hwpt_page_response {
	780	+ __u32 cookie;
	781	+ __u32 code;
	782	+};
	783	+
	784	+/**
	785	+ * struct iommu_fault_alloc - ioctl(IOMMU_FAULT_QUEUE_ALLOC)
	786	+ * @size: sizeof(struct iommu_fault_alloc)
	787	+ * @flags: Must be 0
	788	+ * @out_fault_id: The ID of the new FAULT
	789	+ * @out_fault_fd: The fd of the new FAULT
	790	+ *
	791	+ * Explicitly allocate a fault handling object.
	792	+ */
	793	+struct iommu_fault_alloc {
	794	+ __u32 size;
	795	+ __u32 flags;
	796	+ __u32 out_fault_id;
	797	+ __u32 out_fault_fd;
	798	+};
	799	+#define IOMMU_FAULT_QUEUE_ALLOC _IO(IOMMUFD_TYPE, IOMMUFD_CMD_FAULT_QUEUE_ALLOC)
695	800	#endif

tools/testing/selftests/iommu/iommufd.c+63−23

@@ -279,6 +279,9 @@ TEST_F(iommufd_ioas, alloc_hwpt_nested) · 278 unmodified lines
279	279	uint32_t parent_hwpt_id = 0;
280	280	uint32_t parent_hwpt_id_not_work = 0;
281	281	uint32_t test_hwpt_id = 0;
	282	+ uint32_t iopf_hwpt_id;
	283	+ uint32_t fault_id;
	284	+ uint32_t fault_fd;
282	285
283	286	if (self->device_id) {
284	287	/* Negative tests */
@@ -326,6 +329,7 @@ TEST_F(iommufd_ioas, alloc_hwpt_nested) · 41 unmodified lines
326	329	sizeof(data));
327	330
328	331	/* Allocate two nested hwpts sharing one common parent hwpt */
	332	+ test_ioctl_fault_alloc(&fault_id, &fault_fd);
329	333	test_cmd_hwpt_alloc_nested(self->device_id, parent_hwpt_id, 0,
330	334	&nested_hwpt_id[0],
331	335	IOMMU_HWPT_DATA_SELFTEST, &data,
@@ -334,6 +338,14 @@ TEST_F(iommufd_ioas, alloc_hwpt_nested) · 2 unmodified lines
334	338	&nested_hwpt_id[1],
335	339	IOMMU_HWPT_DATA_SELFTEST, &data,
336	340	sizeof(data));
	341	+ test_err_hwpt_alloc_iopf(ENOENT, self->device_id, parent_hwpt_id,
	342	+ UINT32_MAX, IOMMU_HWPT_FAULT_ID_VALID,
	343	+ &iopf_hwpt_id, IOMMU_HWPT_DATA_SELFTEST,
	344	+ &data, sizeof(data));
	345	+ test_cmd_hwpt_alloc_iopf(self->device_id, parent_hwpt_id, fault_id,
	346	+ IOMMU_HWPT_FAULT_ID_VALID, &iopf_hwpt_id,
	347	+ IOMMU_HWPT_DATA_SELFTEST, &data,
	348	+ sizeof(data));
337	349	test_cmd_hwpt_check_iotlb_all(nested_hwpt_id[0],
338	350	IOMMU_TEST_IOTLB_DEFAULT);
339	351	test_cmd_hwpt_check_iotlb_all(nested_hwpt_id[1],
@@ -504,14 +516,24 @@ TEST_F(iommufd_ioas, alloc_hwpt_nested) · 164 unmodified lines
504	516	_test_ioctl_destroy(self->fd, nested_hwpt_id[1]));
505	517	test_ioctl_destroy(nested_hwpt_id[0]);
506	518
	519	+ /* Switch from nested_hwpt_id[1] to iopf_hwpt_id */
	520	+ test_cmd_mock_domain_replace(self->stdev_id, iopf_hwpt_id);
	521	+ EXPECT_ERRNO(EBUSY,
	522	+ _test_ioctl_destroy(self->fd, iopf_hwpt_id));
	523	+ /* Trigger an IOPF on the device */
	524	+ test_cmd_trigger_iopf(self->device_id, fault_fd);
	525	+
507	526	/* Detach from nested_hwpt_id[1] and destroy it */
508	527	test_cmd_mock_domain_replace(self->stdev_id, parent_hwpt_id);
509	528	test_ioctl_destroy(nested_hwpt_id[1]);
	529	+ test_ioctl_destroy(iopf_hwpt_id);
510	530
511	531	/* Detach from the parent hw_pagetable and destroy it */
512	532	test_cmd_mock_domain_replace(self->stdev_id, self->ioas_id);
513	533	test_ioctl_destroy(parent_hwpt_id);
514	534	test_ioctl_destroy(parent_hwpt_id_not_work);
	535	+ close(fault_fd);
	536	+ test_ioctl_destroy(fault_id);
515	537	} else {
516	538	test_err_hwpt_alloc(ENOENT, self->device_id, self->ioas_id, 0,
517	539	&parent_hwpt_id);
@@ -1722,10 +1744,17 @@ FIXTURE_VARIANT(iommufd_dirty_tracking) · 1204 unmodified lines
1722	1744
1723	1745	FIXTURE_SETUP(iommufd_dirty_tracking)
1724	1746	{
	1747	+ unsigned long size;
1725	1748	int mmap_flags;
1726	1749	void *vrc;
1727	1750	int rc;
1728	1751
	1752	+ if (variant->buffer_size < MOCK_PAGE_SIZE) {
	1753	+ SKIP(return,
	1754	+ "Skipping buffer_size=%lu, less than MOCK_PAGE_SIZE=%lu",
	1755	+ variant->buffer_size, MOCK_PAGE_SIZE);
	1756	+ }
	1757	+
1729	1758	self->fd = open("/dev/iommu", O_RDWR);
1730	1759	ASSERT_NE(-1, self->fd);
1731	1760
@@ -1749,12 +1778,11 @@ FIXTURE_SETUP(iommufd_dirty_tracking) · 17 unmodified lines
1749	1778	assert(vrc == self->buffer);
1750	1779
1751	1780	self->page_size = MOCK_PAGE_SIZE;
1752		− self->bitmap_size =
1753		− variant->buffer_size / self->page_size / BITS_PER_BYTE;
	1781	+ self->bitmap_size = variant->buffer_size / self->page_size;
1754	1782
1755	1783	/* Provision with an extra (PAGE_SIZE) for the unaligned case */
1756		− rc = posix_memalign(&self->bitmap, PAGE_SIZE,
1757		− self->bitmap_size + PAGE_SIZE);
	1784	+ size = DIV_ROUND_UP(self->bitmap_size, BITS_PER_BYTE);
	1785	+ rc = posix_memalign(&self->bitmap, PAGE_SIZE, size + PAGE_SIZE);
1758	1786	assert(!rc);
1759	1787	assert(self->bitmap);
1760	1788	assert((uintptr_t)self->bitmap % PAGE_SIZE == 0);
@@ -1775,51 +1803,63 @@ FIXTURE_SETUP(iommufd_dirty_tracking) · 14 unmodified lines
1775	1803	FIXTURE_TEARDOWN(iommufd_dirty_tracking)
1776	1804	{
1777	1805	munmap(self->buffer, variant->buffer_size);
1778		− munmap(self->bitmap, self->bitmap_size);
	1806	+ munmap(self->bitmap, DIV_ROUND_UP(self->bitmap_size, BITS_PER_BYTE));
1779	1807	teardown_iommufd(self->fd, _metadata);
1780	1808	}
1781	1809
1782		−FIXTURE_VARIANT_ADD(iommufd_dirty_tracking, domain_dirty128k)
	1810	+FIXTURE_VARIANT_ADD(iommufd_dirty_tracking, domain_dirty8k)
	1811	+{
	1812	+ /* half of an u8 index bitmap */
	1813	+ .buffer_size = 8UL * 1024UL,
	1814	+};
	1815	+
	1816	+FIXTURE_VARIANT_ADD(iommufd_dirty_tracking, domain_dirty16k)
	1817	+{
	1818	+ /* one u8 index bitmap */
	1819	+ .buffer_size = 16UL * 1024UL,
	1820	+};
	1821	+
	1822	+FIXTURE_VARIANT_ADD(iommufd_dirty_tracking, domain_dirty64k)
1783	1823	{
1784	1824	/* one u32 index bitmap */
1785		− .buffer_size = 128UL * 1024UL,
	1825	+ .buffer_size = 64UL * 1024UL,
1786	1826	};
1787	1827
1788		−FIXTURE_VARIANT_ADD(iommufd_dirty_tracking, domain_dirty256k)
	1828	+FIXTURE_VARIANT_ADD(iommufd_dirty_tracking, domain_dirty128k)
1789	1829	{
1790	1830	/* one u64 index bitmap */
1791		− .buffer_size = 256UL * 1024UL,
	1831	+ .buffer_size = 128UL * 1024UL,
1792	1832	};
1793	1833
1794		−FIXTURE_VARIANT_ADD(iommufd_dirty_tracking, domain_dirty640k)
	1834	+FIXTURE_VARIANT_ADD(iommufd_dirty_tracking, domain_dirty320k)
1795	1835	{
1796	1836	/* two u64 index and trailing end bitmap */
1797		− .buffer_size = 640UL * 1024UL,
	1837	+ .buffer_size = 320UL * 1024UL,
1798	1838	};
1799	1839
1800		−FIXTURE_VARIANT_ADD(iommufd_dirty_tracking, domain_dirty128M)
	1840	+FIXTURE_VARIANT_ADD(iommufd_dirty_tracking, domain_dirty64M)
1801	1841	{
1802		− /* 4K bitmap (128M IOVA range) */
1803		− .buffer_size = 128UL * 1024UL * 1024UL,
	1842	+ /* 4K bitmap (64M IOVA range) */
	1843	+ .buffer_size = 64UL * 1024UL * 1024UL,
1804	1844	};
1805	1845
1806		−FIXTURE_VARIANT_ADD(iommufd_dirty_tracking, domain_dirty128M_huge)
	1846	+FIXTURE_VARIANT_ADD(iommufd_dirty_tracking, domain_dirty64M_huge)
1807	1847	{
1808		− /* 4K bitmap (128M IOVA range) */
1809		− .buffer_size = 128UL * 1024UL * 1024UL,
	1848	+ /* 4K bitmap (64M IOVA range) */
	1849	+ .buffer_size = 64UL * 1024UL * 1024UL,
1810	1850	.hugepages = true,
1811	1851	};
1812	1852
1813		−FIXTURE_VARIANT_ADD(iommufd_dirty_tracking, domain_dirty256M)
	1853	+FIXTURE_VARIANT_ADD(iommufd_dirty_tracking, domain_dirty128M)
1814	1854	{
1815		− /* 8K bitmap (256M IOVA range) */
1816		− .buffer_size = 256UL * 1024UL * 1024UL,
	1855	+ /* 8K bitmap (128M IOVA range) */
	1856	+ .buffer_size = 128UL * 1024UL * 1024UL,
1817	1857	};
1818	1858
1819		−FIXTURE_VARIANT_ADD(iommufd_dirty_tracking, domain_dirty256M_huge)
	1859	+FIXTURE_VARIANT_ADD(iommufd_dirty_tracking, domain_dirty128M_huge)
1820	1860	{
1821		− /* 8K bitmap (256M IOVA range) */
1822		− .buffer_size = 256UL * 1024UL * 1024UL,
	1861	+ /* 8K bitmap (128M IOVA range) */
	1862	+ .buffer_size = 128UL * 1024UL * 1024UL,
1823	1863	.hugepages = true,
1824	1864	};
1825	1865

tools/testing/selftests/iommu/iommufd_fail_nth.c+1−1

@@ -615,7 +615,7 @@ TEST_FAIL_NTH(basic_fail_nth, device) · 614 unmodified lines
615	615	if (_test_cmd_get_hw_info(self->fd, idev_id, &info, sizeof(info), NULL))
616	616	return -1;
617	617
618		− if (_test_cmd_hwpt_alloc(self->fd, idev_id, ioas_id, 0, &hwpt_id,
	618	+ if (_test_cmd_hwpt_alloc(self->fd, idev_id, ioas_id, 0, 0, &hwpt_id,
619	619	IOMMU_HWPT_DATA_NONE, 0, 0))
620	620	return -1;
621	621

tools/testing/selftests/iommu/iommufd_utils.h+85−7

@@ -22,6 +22,8 @@ · 21 unmodified lines
22	22	#define BIT_MASK(nr) (1UL << ((nr) % __BITS_PER_LONG))
23	23	#define BIT_WORD(nr) ((nr) / __BITS_PER_LONG)
24	24
	25	+#define DIV_ROUND_UP(n, d) (((n) + (d) - 1) / (d))
	26	+
25	27	static inline void set_bit(unsigned int nr, unsigned long *addr)
26	28	{
27	29	unsigned long mask = BIT_MASK(nr);
@@ -153,7 +155,7 @@ static int _test_cmd_mock_domain_replace(int fd, __u32 stdev_id, __u32 pt_id, · 125 unmodified lines
153	155	EXPECT_ERRNO(_errno, _test_cmd_mock_domain_replace(self->fd, stdev_id, \
154	156	pt_id, NULL))
155	157
156		−static int _test_cmd_hwpt_alloc(int fd, __u32 device_id, __u32 pt_id,
	158	+static int _test_cmd_hwpt_alloc(int fd, __u32 device_id, __u32 pt_id, __u32 ft_id,
157	159	__u32 flags, __u32 *hwpt_id, __u32 data_type,
158	160	void *data, size_t data_len)
159	161	{
@@ -165,6 +167,7 @@ static int _test_cmd_hwpt_alloc(int fd, __u32 device_id, __u32 pt_id, · 5 unmodified lines
165	167	.data_type = data_type,
166	168	.data_len = data_len,
167	169	.data_uptr = (uint64_t)data,
	170	+ .fault_id = ft_id,
168	171	};
169	172	int ret;
170	173
@@ -177,24 +180,36 @@ static int _test_cmd_hwpt_alloc(int fd, __u32 device_id, __u32 pt_id, · 6 unmodified lines
177	180	}
178	181
179	182	#define test_cmd_hwpt_alloc(device_id, pt_id, flags, hwpt_id) \
180		− ASSERT_EQ(0, _test_cmd_hwpt_alloc(self->fd, device_id, pt_id, flags, \
	183	+ ASSERT_EQ(0, _test_cmd_hwpt_alloc(self->fd, device_id, pt_id, 0, flags, \
181	184	hwpt_id, IOMMU_HWPT_DATA_NONE, NULL, \
182	185	0))
183	186	#define test_err_hwpt_alloc(_errno, device_id, pt_id, flags, hwpt_id) \
184	187	EXPECT_ERRNO(_errno, _test_cmd_hwpt_alloc( \
185		− self->fd, device_id, pt_id, flags, \
	188	+ self->fd, device_id, pt_id, 0, flags, \
186	189	hwpt_id, IOMMU_HWPT_DATA_NONE, NULL, 0))
187	190
188	191	#define test_cmd_hwpt_alloc_nested(device_id, pt_id, flags, hwpt_id, \
189	192	data_type, data, data_len) \
190		− ASSERT_EQ(0, _test_cmd_hwpt_alloc(self->fd, device_id, pt_id, flags, \
	193	+ ASSERT_EQ(0, _test_cmd_hwpt_alloc(self->fd, device_id, pt_id, 0, flags, \
191	194	hwpt_id, data_type, data, data_len))
192	195	#define test_err_hwpt_alloc_nested(_errno, device_id, pt_id, flags, hwpt_id, \
193	196	data_type, data, data_len) \
194	197	EXPECT_ERRNO(_errno, \
195		− _test_cmd_hwpt_alloc(self->fd, device_id, pt_id, flags, \
	198	+ _test_cmd_hwpt_alloc(self->fd, device_id, pt_id, 0, flags, \
196	199	hwpt_id, data_type, data, data_len))
197	200
	201	+#define test_cmd_hwpt_alloc_iopf(device_id, pt_id, fault_id, flags, hwpt_id, \
	202	+ data_type, data, data_len) \
	203	+ ASSERT_EQ(0, _test_cmd_hwpt_alloc(self->fd, device_id, pt_id, fault_id, \
	204	+ flags, hwpt_id, data_type, data, \
	205	+ data_len))
	206	+#define test_err_hwpt_alloc_iopf(_errno, device_id, pt_id, fault_id, flags, \
	207	+ hwpt_id, data_type, data, data_len) \
	208	+ EXPECT_ERRNO(_errno, \
	209	+ _test_cmd_hwpt_alloc(self->fd, device_id, pt_id, fault_id, \
	210	+ flags, hwpt_id, data_type, data, \
	211	+ data_len))
	212	+
198	213	#define test_cmd_hwpt_check_iotlb(hwpt_id, iotlb_id, expected) \
199	214	({ \
200	215	struct iommu_test_cmd test_cmd = { \
@@ -346,12 +361,12 @@ static int _test_cmd_mock_domain_set_dirty(int fd, __u32 hwpt_id, size_t length, · 145 unmodified lines
346	361	static int _test_mock_dirty_bitmaps(int fd, __u32 hwpt_id, size_t length,
347	362	__u64 iova, size_t page_size,
348	363	size_t pte_page_size, __u64 *bitmap,
349		− __u64 bitmap_size, __u32 flags,
	364	+ __u64 nbits, __u32 flags,
350	365	struct __test_metadata *_metadata)
351	366	{
352	367	unsigned long npte = pte_page_size / page_size, pteset = 2 * npte;
353		− unsigned long nbits = bitmap_size * BITS_PER_BYTE;
354	368	unsigned long j, i, nr = nbits / pteset ?: 1;
	369	+ unsigned long bitmap_size = DIV_ROUND_UP(nbits, BITS_PER_BYTE);
355	370	__u64 out_dirty = 0;
356	371
357	372	/* Mark all even bits as dirty in the mock domain */
@@ -684,3 +699,66 @@ static int _test_cmd_get_hw_info(int fd, __u32 device_id, void *data, · 326 unmodified lines
684	699
685	700	#define test_cmd_get_hw_capabilities(device_id, caps, mask) \
686	701	ASSERT_EQ(0, _test_cmd_get_hw_info(self->fd, device_id, NULL, 0, &caps))
	702	+
	703	+static int _test_ioctl_fault_alloc(int fd, __u32 fault_id, __u32 fault_fd)
	704	+{
	705	+ struct iommu_fault_alloc cmd = {
	706	+ .size = sizeof(cmd),
	707	+ };
	708	+ int ret;
	709	+
	710	+ ret = ioctl(fd, IOMMU_FAULT_QUEUE_ALLOC, &cmd);
	711	+ if (ret)
	712	+ return ret;
	713	+ *fault_id = cmd.out_fault_id;
	714	+ *fault_fd = cmd.out_fault_fd;
	715	+ return 0;
	716	+}
	717	+
	718	+#define test_ioctl_fault_alloc(fault_id, fault_fd) \
	719	+ ({ \
	720	+ ASSERT_EQ(0, _test_ioctl_fault_alloc(self->fd, fault_id, \
	721	+ fault_fd)); \
	722	+ ASSERT_NE(0, *(fault_id)); \
	723	+ ASSERT_NE(0, *(fault_fd)); \
	724	+ })
	725	+
	726	+static int _test_cmd_trigger_iopf(int fd, __u32 device_id, __u32 fault_fd)
	727	+{
	728	+ struct iommu_test_cmd trigger_iopf_cmd = {
	729	+ .size = sizeof(trigger_iopf_cmd),
	730	+ .op = IOMMU_TEST_OP_TRIGGER_IOPF,
	731	+ .trigger_iopf = {
	732	+ .dev_id = device_id,
	733	+ .pasid = 0x1,
	734	+ .grpid = 0x2,
	735	+ .perm = IOMMU_PGFAULT_PERM_READ \| IOMMU_PGFAULT_PERM_WRITE,
	736	+ .addr = 0xdeadbeaf,
	737	+ },
	738	+ };
	739	+ struct iommu_hwpt_page_response response = {
	740	+ .code = IOMMUFD_PAGE_RESP_SUCCESS,
	741	+ };
	742	+ struct iommu_hwpt_pgfault fault = {};
	743	+ ssize_t bytes;
	744	+ int ret;
	745	+
	746	+ ret = ioctl(fd, _IOMMU_TEST_CMD(IOMMU_TEST_OP_TRIGGER_IOPF), &trigger_iopf_cmd);
	747	+ if (ret)
	748	+ return ret;
	749	+
	750	+ bytes = read(fault_fd, &fault, sizeof(fault));
	751	+ if (bytes <= 0)
	752	+ return -EIO;
	753	+
	754	+ response.cookie = fault.cookie;
	755	+
	756	+ bytes = write(fault_fd, &response, sizeof(response));
	757	+ if (bytes <= 0)
	758	+ return -EIO;
	759	+
	760	+ return 0;
	761	+}
	762	+
	763	+#define test_cmd_trigger_iopf(device_id, fault_fd) \
	764	+ ASSERT_EQ(0, _test_cmd_trigger_iopf(self->fd, device_id, fault_fd))