Every IOMMU driver should be able to implement the needed iommu domain ops to control dirty tracking. Connect a hw_pagetable to the IOMMU core dirty tracking ops, specifically the ability to enable/disable dirty tracking on an IOMMU domain (hw_pagetable id). To that end add an io_pagetable kernel API to toggle dirty tracking: * iopt_set_dirty_tracking(iopt, [domain], state) The intended caller of this is via the hw_pagetable object that is created. Internally it will ensure the leftover dirty state is cleared /right before/ dirty tracking starts. This is also useful for iommu drivers which may decide that dirty tracking is always-enabled at boot without wanting to toggle dynamically via corresponding iommu domain op.
Unified · Split
| @@ -196,3 +196,27 @@ out_put_idev: · 195 unmodified lines | ||
| 196 | 196 | iommufd_put_object(&idev->obj); |
| 197 | 197 | return rc; |
| 198 | 198 | } |
| 199 | + | |
| 200 | +int iommufd_hwpt_set_dirty_tracking(struct iommufd_ucmd *ucmd) | |
| 201 | +{ | |
| 202 | + struct iommu_hwpt_set_dirty_tracking *cmd = ucmd->cmd; | |
| 203 | + struct iommufd_hw_pagetable *hwpt; | |
| 204 | + struct iommufd_ioas *ioas; | |
| 205 | + int rc = -EOPNOTSUPP; | |
| 206 | + bool enable; | |
| 207 | + | |
| 208 | + if (cmd->flags & ~IOMMU_HWPT_DIRTY_TRACKING_ENABLE) | |
| 209 | + return rc; | |
| 210 | + | |
| 211 | + hwpt = iommufd_get_hwpt(ucmd, cmd->hwpt_id); | |
| 212 | + if (IS_ERR(hwpt)) | |
| 213 | + return PTR_ERR(hwpt); | |
| 214 | + | |
| 215 | + ioas = hwpt->ioas; | |
| 216 | + enable = cmd->flags & IOMMU_HWPT_DIRTY_TRACKING_ENABLE; | |
| 217 | + | |
| 218 | + rc = iopt_set_dirty_tracking(&ioas->iopt, hwpt->domain, enable); | |
| 219 | + | |
| 220 | + iommufd_put_object(&hwpt->obj); | |
| 221 | + return rc; | |
| 222 | +} | |
| @@ -412,6 +412,60 @@ int iopt_map_user_pages(struct iommufd_ctx *ictx, struct io_pagetable *iopt, · 411 unmodified lines | ||
| 412 | 412 | return 0; |
| 413 | 413 | } |
| 414 | 414 | |
| 415 | +static int iopt_clear_dirty_data(struct io_pagetable *iopt, | |
| 416 | + struct iommu_domain *domain) | |
| 417 | +{ | |
| 418 | + const struct iommu_dirty_ops *ops = domain->dirty_ops; | |
| 419 | + struct iommu_iotlb_gather gather; | |
| 420 | + struct iommu_dirty_bitmap dirty; | |
| 421 | + struct iopt_area *area; | |
| 422 | + int ret = 0; | |
| 423 | + | |
| 424 | + lockdep_assert_held_read(&iopt->iova_rwsem); | |
| 425 | + | |
| 426 | + iommu_dirty_bitmap_init(&dirty, NULL, &gather); | |
| 427 | + | |
| 428 | + for (area = iopt_area_iter_first(iopt, 0, ULONG_MAX); area; | |
| 429 | + area = iopt_area_iter_next(area, 0, ULONG_MAX)) { | |
| 430 | + if (!area->pages) | |
| 431 | + continue; | |
| 432 | + | |
| 433 | + ret = ops->read_and_clear_dirty(domain, iopt_area_iova(area), | |
| 434 | + iopt_area_length(area), 0, | |
| 435 | + &dirty); | |
| 436 | + if (ret) | |
| 437 | + break; | |
| 438 | + } | |
| 439 | + | |
| 440 | + iommu_iotlb_sync(domain, &gather); | |
| 441 | + return ret; | |
| 442 | +} | |
| 443 | + | |
| 444 | +int iopt_set_dirty_tracking(struct io_pagetable *iopt, | |
| 445 | + struct iommu_domain *domain, bool enable) | |
| 446 | +{ | |
| 447 | + const struct iommu_dirty_ops *ops = domain->dirty_ops; | |
| 448 | + int ret = 0; | |
| 449 | + | |
| 450 | + if (!ops) | |
| 451 | + return -EOPNOTSUPP; | |
| 452 | + | |
| 453 | + down_read(&iopt->iova_rwsem); | |
| 454 | + | |
| 455 | + /* Clear dirty bits from PTEs to ensure a clean snapshot */ | |
| 456 | + if (enable) { | |
| 457 | + ret = iopt_clear_dirty_data(iopt, domain); | |
| 458 | + if (ret) | |
| 459 | + goto out_unlock; | |
| 460 | + } | |
| 461 | + | |
| 462 | + ret = ops->set_dirty_tracking(domain, enable); | |
| 463 | + | |
| 464 | +out_unlock: | |
| 465 | + up_read(&iopt->iova_rwsem); | |
| 466 | + return ret; | |
| 467 | +} | |
| 468 | + | |
| 415 | 469 | int iopt_get_pages(struct io_pagetable *iopt, unsigned long iova, |
| 416 | 470 | unsigned long length, struct list_head *pages_list) |
| 417 | 471 | { |
| @@ -8,6 +8,7 @@ · 7 unmodified lines | ||
| 8 | 8 | #include <linux/xarray.h> |
| 9 | 9 | #include <linux/refcount.h> |
| 10 | 10 | #include <linux/uaccess.h> |
| 11 | +#include <uapi/linux/iommufd.h> | |
| 11 | 12 | |
| 12 | 13 | struct iommu_domain; |
| 13 | 14 | struct iommu_group; |
| @@ -70,6 +71,9 @@ int iopt_unmap_iova(struct io_pagetable *iopt, unsigned long iova, · 56 unmodified lines | ||
| 70 | 71 | unsigned long length, unsigned long *unmapped); |
| 71 | 72 | int iopt_unmap_all(struct io_pagetable *iopt, unsigned long *unmapped); |
| 72 | 73 | |
| 74 | +int iopt_set_dirty_tracking(struct io_pagetable *iopt, | |
| 75 | + struct iommu_domain *domain, bool enable); | |
| 76 | + | |
| 73 | 77 | void iommufd_access_notify_unmap(struct io_pagetable *iopt, unsigned long iova, |
| 74 | 78 | unsigned long length); |
| 75 | 79 | int iopt_table_add_domain(struct io_pagetable *iopt, |
| @@ -240,6 +244,14 @@ struct iommufd_hw_pagetable { · 164 unmodified lines | ||
| 240 | 244 | struct list_head hwpt_item; |
| 241 | 245 | }; |
| 242 | 246 | |
| 247 | +static inline struct iommufd_hw_pagetable * | |
| 248 | +iommufd_get_hwpt(struct iommufd_ucmd *ucmd, u32 id) | |
| 249 | +{ | |
| 250 | + return container_of(iommufd_get_object(ucmd->ictx, id, | |
| 251 | + IOMMUFD_OBJ_HW_PAGETABLE), | |
| 252 | + struct iommufd_hw_pagetable, obj); | |
| 253 | +} | |
| 254 | +int iommufd_hwpt_set_dirty_tracking(struct iommufd_ucmd *ucmd); | |
| 243 | 255 | struct iommufd_hw_pagetable * |
| 244 | 256 | iommufd_hw_pagetable_alloc(struct iommufd_ctx *ictx, struct iommufd_ioas *ioas, |
| 245 | 257 | struct iommufd_device *idev, u32 flags, |
| @@ -307,6 +307,7 @@ union ucmd_buffer { · 306 unmodified lines | ||
| 307 | 307 | struct iommu_destroy destroy; |
| 308 | 308 | struct iommu_hw_info info; |
| 309 | 309 | struct iommu_hwpt_alloc hwpt; |
| 310 | + struct iommu_hwpt_set_dirty_tracking set_dirty_tracking; | |
| 310 | 311 | struct iommu_ioas_alloc alloc; |
| 311 | 312 | struct iommu_ioas_allow_iovas allow_iovas; |
| 312 | 313 | struct iommu_ioas_copy ioas_copy; |
| @@ -342,6 +343,8 @@ static const struct iommufd_ioctl_op iommufd_ioctl_ops[] = { · 29 unmodified lines | ||
| 342 | 343 | __reserved), |
| 343 | 344 | IOCTL_OP(IOMMU_HWPT_ALLOC, iommufd_hwpt_alloc, struct iommu_hwpt_alloc, |
| 344 | 345 | __reserved), |
| 346 | + IOCTL_OP(IOMMU_HWPT_SET_DIRTY_TRACKING, iommufd_hwpt_set_dirty_tracking, | |
| 347 | + struct iommu_hwpt_set_dirty_tracking, __reserved), | |
| 345 | 348 | IOCTL_OP(IOMMU_IOAS_ALLOC, iommufd_ioas_alloc_ioctl, |
| 346 | 349 | struct iommu_ioas_alloc, out_ioas_id), |
| 347 | 350 | IOCTL_OP(IOMMU_IOAS_ALLOW_IOVAS, iommufd_ioas_allow_iovas, |
| @@ -47,6 +47,7 @@ enum { · 46 unmodified lines | ||
| 47 | 47 | IOMMUFD_CMD_VFIO_IOAS, |
| 48 | 48 | IOMMUFD_CMD_HWPT_ALLOC, |
| 49 | 49 | IOMMUFD_CMD_GET_HW_INFO, |
| 50 | + IOMMUFD_CMD_HWPT_SET_DIRTY_TRACKING, | |
| 50 | 51 | }; |
| 51 | 52 | |
| 52 | 53 | /** |
| @@ -453,4 +454,31 @@ struct iommu_hw_info { · 400 unmodified lines | ||
| 453 | 454 | __u32 __reserved; |
| 454 | 455 | }; |
| 455 | 456 | #define IOMMU_GET_HW_INFO _IO(IOMMUFD_TYPE, IOMMUFD_CMD_GET_HW_INFO) |
| 457 | + | |
| 458 | +/* | |
| 459 | + * enum iommufd_hwpt_set_dirty_tracking_flags - Flags for steering dirty | |
| 460 | + * tracking | |
| 461 | + * @IOMMU_HWPT_DIRTY_TRACKING_ENABLE: Enable dirty tracking | |
| 462 | + */ | |
| 463 | +enum iommufd_hwpt_set_dirty_tracking_flags { | |
| 464 | + IOMMU_HWPT_DIRTY_TRACKING_ENABLE = 1, | |
| 465 | +}; | |
| 466 | + | |
| 467 | +/** | |
| 468 | + * struct iommu_hwpt_set_dirty_tracking - ioctl(IOMMU_HWPT_SET_DIRTY_TRACKING) | |
| 469 | + * @size: sizeof(struct iommu_hwpt_set_dirty_tracking) | |
| 470 | + * @flags: Combination of enum iommufd_hwpt_set_dirty_tracking_flags | |
| 471 | + * @hwpt_id: HW pagetable ID that represents the IOMMU domain | |
| 472 | + * @__reserved: Must be 0 | |
| 473 | + * | |
| 474 | + * Toggle dirty tracking on an HW pagetable. | |
| 475 | + */ | |
| 476 | +struct iommu_hwpt_set_dirty_tracking { | |
| 477 | + __u32 size; | |
| 478 | + __u32 flags; | |
| 479 | + __u32 hwpt_id; | |
| 480 | + __u32 __reserved; | |
| 481 | +}; | |
| 482 | +#define IOMMU_HWPT_SET_DIRTY_TRACKING _IO(IOMMUFD_TYPE, \ | |
| 483 | + IOMMUFD_CMD_HWPT_SET_DIRTY_TRACKING) | |
| 456 | 484 | #endif |