| @@ -73,6 +73,7 @@ struct vfio_iommu { · 72 unmodified lines |
| 73 | 73 | bool v2; |
| 74 | 74 | bool nesting; |
| 75 | 75 | bool dirty_page_tracking; |
| 76 | + bool pinned_page_dirty_scope; |
| 76 | 77 | }; |
| 77 | 78 | |
| 78 | 79 | struct vfio_domain { |
| @@ -100,6 +101,7 @@ struct vfio_group { · 21 unmodified lines |
| 100 | 101 | struct iommu_group *iommu_group; |
| 101 | 102 | struct list_head next; |
| 102 | 103 | bool mdev_group; /* An mdev group */ |
| 104 | + bool pinned_page_dirty_scope; |
| 103 | 105 | }; |
| 104 | 106 | |
| 105 | 107 | struct vfio_iova { |
| @@ -143,6 +145,10 @@ struct vfio_regions { · 37 unmodified lines |
| 143 | 145 | |
| 144 | 146 | static int put_pfn(unsigned long pfn, int prot); |
| 145 | 147 | |
| 148 | +static struct vfio_group *vfio_iommu_find_iommu_group(struct vfio_iommu *iommu, |
| 149 | + struct iommu_group *iommu_group); |
| 150 | + |
| 151 | +static void update_pinned_page_dirty_scope(struct vfio_iommu *iommu); |
| 146 | 152 | /* |
| 147 | 153 | * This code handles mapping and unmapping of user data buffers |
| 148 | 154 | * into DMA'ble space using the IOMMU |
| @@ -622,11 +628,13 @@ static int vfio_unpin_page_external(struct vfio_dma *dma, dma_addr_t iova, · 473 unmodified lines |
| 622 | 628 | } |
| 623 | 629 | |
| 624 | 630 | static int vfio_iommu_type1_pin_pages(void *iommu_data, |
| 631 | + struct iommu_group *iommu_group, |
| 625 | 632 | unsigned long *user_pfn, |
| 626 | 633 | int npage, int prot, |
| 627 | 634 | unsigned long *phys_pfn) |
| 628 | 635 | { |
| 629 | 636 | struct vfio_iommu *iommu = iommu_data; |
| 637 | + struct vfio_group *group; |
| 630 | 638 | int i, j, ret; |
| 631 | 639 | unsigned long remote_vaddr; |
| 632 | 640 | struct vfio_dma *dma; |
| @@ -699,8 +707,14 @@ static int vfio_iommu_type1_pin_pages(void *iommu_data, · 66 unmodified lines |
| 699 | 707 | (iova - dma->iova) >> pgshift, 1); |
| 700 | 708 | } |
| 701 | 709 | } |
| 702 | | − |
| 703 | 710 | ret = i; |
| 711 | + |
| 712 | + group = vfio_iommu_find_iommu_group(iommu, iommu_group); |
| 713 | + if (!group->pinned_page_dirty_scope) { |
| 714 | + group->pinned_page_dirty_scope = true; |
| 715 | + update_pinned_page_dirty_scope(iommu); |
| 716 | + } |
| 717 | + |
| 704 | 718 | goto pin_done; |
| 705 | 719 | |
| 706 | 720 | pin_unwind: |
| @@ -960,8 +974,9 @@ static void vfio_update_pgsize_bitmap(struct vfio_iommu *iommu) · 253 unmodified lines |
| 960 | 974 | } |
| 961 | 975 | } |
| 962 | 976 | |
| 963 | | −static int update_user_bitmap(u64 __user *bitmap, struct vfio_dma *dma, |
| 964 | | − dma_addr_t base_iova, size_t pgsize) |
| 977 | +static int update_user_bitmap(u64 __user *bitmap, struct vfio_iommu *iommu, |
| 978 | + struct vfio_dma *dma, dma_addr_t base_iova, |
| 979 | + size_t pgsize) |
| 965 | 980 | { |
| 966 | 981 | unsigned long pgshift = __ffs(pgsize); |
| 967 | 982 | unsigned long nbits = dma->size >> pgshift; |
| @@ -970,8 +985,11 @@ static int update_user_bitmap(u64 __user *bitmap, struct vfio_dma *dma, · 2 unmodified lines |
| 970 | 985 | unsigned long shift = bit_offset % BITS_PER_LONG; |
| 971 | 986 | unsigned long leftover; |
| 972 | 987 | |
| 973 | | − /* mark all pages dirty if all pages are pinned and mapped. */ |
| 974 | | − if (dma->iommu_mapped) |
| 988 | + /* |
| 989 | + * mark all pages dirty if any IOMMU capable device is not able |
| 990 | + * to report dirty pages and all pages are pinned and mapped. |
| 991 | + */ |
| 992 | + if (!iommu->pinned_page_dirty_scope && dma->iommu_mapped) |
| 975 | 993 | bitmap_set(dma->bitmap, 0, nbits); |
| 976 | 994 | |
| 977 | 995 | if (shift) { |
| @@ -1024,7 +1042,7 @@ static int vfio_iova_dirty_bitmap(u64 __user *bitmap, struct vfio_iommu *iommu, · 46 unmodified lines |
| 1024 | 1042 | if (dma->iova > iova + size - 1) |
| 1025 | 1043 | break; |
| 1026 | 1044 | |
| 1027 | | − ret = update_user_bitmap(bitmap, dma, iova, pgsize); |
| 1045 | + ret = update_user_bitmap(bitmap, iommu, dma, iova, pgsize); |
| 1028 | 1046 | if (ret) |
| 1029 | 1047 | return ret; |
| 1030 | 1048 | |
| @@ -1169,7 +1187,7 @@ again: · 138 unmodified lines |
| 1169 | 1187 | } |
| 1170 | 1188 | |
| 1171 | 1189 | if (unmap->flags & VFIO_DMA_UNMAP_FLAG_GET_DIRTY_BITMAP) { |
| 1172 | | − ret = update_user_bitmap(bitmap->data, dma, |
| 1190 | + ret = update_user_bitmap(bitmap->data, iommu, dma, |
| 1173 | 1191 | unmap->iova, pgsize); |
| 1174 | 1192 | if (ret) |
| 1175 | 1193 | break; |
| @@ -1521,6 +1539,51 @@ static struct vfio_group *find_iommu_group(struct vfio_domain *domain, · 345 unmodified lines |
| 1521 | 1539 | return NULL; |
| 1522 | 1540 | } |
| 1523 | 1541 | |
| 1542 | +static struct vfio_group *vfio_iommu_find_iommu_group(struct vfio_iommu *iommu, |
| 1543 | + struct iommu_group *iommu_group) |
| 1544 | +{ |
| 1545 | + struct vfio_domain *domain; |
| 1546 | + struct vfio_group *group = NULL; |
| 1547 | + |
| 1548 | + list_for_each_entry(domain, &iommu->domain_list, next) { |
| 1549 | + group = find_iommu_group(domain, iommu_group); |
| 1550 | + if (group) |
| 1551 | + return group; |
| 1552 | + } |
| 1553 | + |
| 1554 | + if (iommu->external_domain) |
| 1555 | + group = find_iommu_group(iommu->external_domain, iommu_group); |
| 1556 | + |
| 1557 | + return group; |
| 1558 | +} |
| 1559 | + |
| 1560 | +static void update_pinned_page_dirty_scope(struct vfio_iommu *iommu) |
| 1561 | +{ |
| 1562 | + struct vfio_domain *domain; |
| 1563 | + struct vfio_group *group; |
| 1564 | + |
| 1565 | + list_for_each_entry(domain, &iommu->domain_list, next) { |
| 1566 | + list_for_each_entry(group, &domain->group_list, next) { |
| 1567 | + if (!group->pinned_page_dirty_scope) { |
| 1568 | + iommu->pinned_page_dirty_scope = false; |
| 1569 | + return; |
| 1570 | + } |
| 1571 | + } |
| 1572 | + } |
| 1573 | + |
| 1574 | + if (iommu->external_domain) { |
| 1575 | + domain = iommu->external_domain; |
| 1576 | + list_for_each_entry(group, &domain->group_list, next) { |
| 1577 | + if (!group->pinned_page_dirty_scope) { |
| 1578 | + iommu->pinned_page_dirty_scope = false; |
| 1579 | + return; |
| 1580 | + } |
| 1581 | + } |
| 1582 | + } |
| 1583 | + |
| 1584 | + iommu->pinned_page_dirty_scope = true; |
| 1585 | +} |
| 1586 | + |
| 1524 | 1587 | static bool vfio_iommu_has_sw_msi(struct list_head *group_resv_regions, |
| 1525 | 1588 | phys_addr_t *base) |
| 1526 | 1589 | { |
| @@ -1928,6 +1991,16 @@ static int vfio_iommu_type1_attach_group(void *iommu_data, · 401 unmodified lines |
| 1928 | 1991 | |
| 1929 | 1992 | list_add(&group->next, |
| 1930 | 1993 | &iommu->external_domain->group_list); |
| 1994 | + /* |
| 1995 | + * Non-iommu backed group cannot dirty memory directly, |
| 1996 | + * it can only use interfaces that provide dirty |
| 1997 | + * tracking. |
| 1998 | + * The iommu scope can only be promoted with the |
| 1999 | + * addition of a dirty tracking group. |
| 2000 | + */ |
| 2001 | + group->pinned_page_dirty_scope = true; |
| 2002 | + if (!iommu->pinned_page_dirty_scope) |
| 2003 | + update_pinned_page_dirty_scope(iommu); |
| 1931 | 2004 | mutex_unlock(&iommu->lock); |
| 1932 | 2005 | |
| 1933 | 2006 | return 0; |
| @@ -2051,6 +2124,13 @@ static int vfio_iommu_type1_attach_group(void *iommu_data, · 117 unmodified lines |
| 2051 | 2124 | done: |
| 2052 | 2125 | /* Delete the old one and insert new iova list */ |
| 2053 | 2126 | vfio_iommu_iova_insert_copy(iommu, &iova_copy); |
| 2127 | + |
| 2128 | + /* |
| 2129 | + * An iommu backed group can dirty memory directly and therefore |
| 2130 | + * demotes the iommu scope until it declares itself dirty tracking |
| 2131 | + * capable via the page pinning interface. |
| 2132 | + */ |
| 2133 | + iommu->pinned_page_dirty_scope = false; |
| 2054 | 2134 | mutex_unlock(&iommu->lock); |
| 2055 | 2135 | vfio_iommu_resv_free(&group_resv_regions); |
| 2056 | 2136 | |
| @@ -2203,6 +2283,7 @@ static void vfio_iommu_type1_detach_group(void *iommu_data, · 146 unmodified lines |
| 2203 | 2283 | struct vfio_iommu *iommu = iommu_data; |
| 2204 | 2284 | struct vfio_domain *domain; |
| 2205 | 2285 | struct vfio_group *group; |
| 2286 | + bool update_dirty_scope = false; |
| 2206 | 2287 | LIST_HEAD(iova_copy); |
| 2207 | 2288 | |
| 2208 | 2289 | mutex_lock(&iommu->lock); |
| @@ -2210,6 +2291,7 @@ static void vfio_iommu_type1_detach_group(void *iommu_data, · 1 unmodified line |
| 2210 | 2291 | if (iommu->external_domain) { |
| 2211 | 2292 | group = find_iommu_group(iommu->external_domain, iommu_group); |
| 2212 | 2293 | if (group) { |
| 2294 | + update_dirty_scope = !group->pinned_page_dirty_scope; |
| 2213 | 2295 | list_del(&group->next); |
| 2214 | 2296 | kfree(group); |
| 2215 | 2297 | |
| @@ -2239,6 +2321,7 @@ static void vfio_iommu_type1_detach_group(void *iommu_data, · 23 unmodified lines |
| 2239 | 2321 | continue; |
| 2240 | 2322 | |
| 2241 | 2323 | vfio_iommu_detach_group(domain, group); |
| 2324 | + update_dirty_scope = !group->pinned_page_dirty_scope; |
| 2242 | 2325 | list_del(&group->next); |
| 2243 | 2326 | kfree(group); |
| 2244 | 2327 | /* |
| @@ -2270,6 +2353,12 @@ static void vfio_iommu_type1_detach_group(void *iommu_data, · 25 unmodified lines |
| 2270 | 2353 | vfio_iommu_iova_free(&iova_copy); |
| 2271 | 2354 | |
| 2272 | 2355 | detach_group_done: |
| 2356 | + /* |
| 2357 | + * Removal of a group without dirty tracking may allow the iommu scope |
| 2358 | + * to be promoted. |
| 2359 | + */ |
| 2360 | + if (update_dirty_scope) |
| 2361 | + update_pinned_page_dirty_scope(iommu); |
| 2273 | 2362 | mutex_unlock(&iommu->lock); |
| 2274 | 2363 | } |
| 2275 | 2364 | |