| @@ -73,6 +73,7 @@ struct vfio_iommu { · 72 unmodified lines |
| 73 | bool v2; | 73 | bool v2; |
| 74 | bool nesting; | 74 | bool nesting; |
| 75 | bool dirty_page_tracking; | 75 | bool dirty_page_tracking; |
| | 76 | bool pinned_page_dirty_scope; |
| 76 | }; | 77 | }; |
| 77 | | 78 | |
| 78 | struct vfio_domain { | 79 | struct vfio_domain { |
| @@ -100,6 +101,7 @@ struct vfio_group { · 21 unmodified lines |
| 100 | struct iommu_group *iommu_group; | 101 | struct iommu_group *iommu_group; |
| 101 | struct list_head next; | 102 | struct list_head next; |
| 102 | bool mdev_group; /* An mdev group */ | 103 | bool mdev_group; /* An mdev group */ |
| | 104 | bool pinned_page_dirty_scope; |
| 103 | }; | 105 | }; |
| 104 | | 106 | |
| 105 | struct vfio_iova { | 107 | struct vfio_iova { |
| @@ -143,6 +145,10 @@ struct vfio_regions { · 37 unmodified lines |
| 143 | | 145 | |
| 144 | static int put_pfn(unsigned long pfn, int prot); | 146 | static int put_pfn(unsigned long pfn, int prot); |
| 145 | | 147 | |
| | 148 | static struct vfio_group *vfio_iommu_find_iommu_group(struct vfio_iommu *iommu, |
| | 149 | struct iommu_group *iommu_group); |
| | 150 | |
| | 151 | static void update_pinned_page_dirty_scope(struct vfio_iommu *iommu); |
| 146 | /* | 152 | /* |
| 147 | * This code handles mapping and unmapping of user data buffers | 153 | * This code handles mapping and unmapping of user data buffers |
| 148 | * into DMA'ble space using the IOMMU | 154 | * into DMA'ble space using the IOMMU |
| @@ -622,11 +628,13 @@ static int vfio_unpin_page_external(struct vfio_dma *dma, dma_addr_t iova, · 473 unmodified lines |
| 622 | } | 628 | } |
| 623 | | 629 | |
| 624 | static int vfio_iommu_type1_pin_pages(void *iommu_data, | 630 | static int vfio_iommu_type1_pin_pages(void *iommu_data, |
| | 631 | struct iommu_group *iommu_group, |
| 625 | unsigned long *user_pfn, | 632 | unsigned long *user_pfn, |
| 626 | int npage, int prot, | 633 | int npage, int prot, |
| 627 | unsigned long *phys_pfn) | 634 | unsigned long *phys_pfn) |
| 628 | { | 635 | { |
| 629 | struct vfio_iommu *iommu = iommu_data; | 636 | struct vfio_iommu *iommu = iommu_data; |
| | 637 | struct vfio_group *group; |
| 630 | int i, j, ret; | 638 | int i, j, ret; |
| 631 | unsigned long remote_vaddr; | 639 | unsigned long remote_vaddr; |
| 632 | struct vfio_dma *dma; | 640 | struct vfio_dma *dma; |
| @@ -699,8 +707,14 @@ static int vfio_iommu_type1_pin_pages(void *iommu_data, · 66 unmodified lines |
| 699 | (iova - dma->iova) >> pgshift, 1); | 707 | (iova - dma->iova) >> pgshift, 1); |
| 700 | } | 708 | } |
| 701 | } | 709 | } |
| 702 | | | |
| 703 | ret = i; | 710 | ret = i; |
| | 711 | |
| | 712 | group = vfio_iommu_find_iommu_group(iommu, iommu_group); |
| | 713 | if (!group->pinned_page_dirty_scope) { |
| | 714 | group->pinned_page_dirty_scope = true; |
| | 715 | update_pinned_page_dirty_scope(iommu); |
| | 716 | } |
| | 717 | |
| 704 | goto pin_done; | 718 | goto pin_done; |
| 705 | | 719 | |
| 706 | pin_unwind: | 720 | pin_unwind: |
| @@ -960,8 +974,9 @@ static void vfio_update_pgsize_bitmap(struct vfio_iommu *iommu) · 253 unmodified lines |
| 960 | } | 974 | } |
| 961 | } | 975 | } |
| 962 | | 976 | |
| 963 | static int update_user_bitmap(u64 __user *bitmap, struct vfio_dma *dma, | 977 | static int update_user_bitmap(u64 __user *bitmap, struct vfio_iommu *iommu, |
| 964 | dma_addr_t base_iova, size_t pgsize) | 978 | struct vfio_dma *dma, dma_addr_t base_iova, |
| | 979 | size_t pgsize) |
| 965 | { | 980 | { |
| 966 | unsigned long pgshift = __ffs(pgsize); | 981 | unsigned long pgshift = __ffs(pgsize); |
| 967 | unsigned long nbits = dma->size >> pgshift; | 982 | unsigned long nbits = dma->size >> pgshift; |
| @@ -970,8 +985,11 @@ static int update_user_bitmap(u64 __user *bitmap, struct vfio_dma *dma, · 2 unmodified lines |
| 970 | unsigned long shift = bit_offset % BITS_PER_LONG; | 985 | unsigned long shift = bit_offset % BITS_PER_LONG; |
| 971 | unsigned long leftover; | 986 | unsigned long leftover; |
| 972 | | 987 | |
| 973 | /* mark all pages dirty if all pages are pinned and mapped. */ | 988 | /* |
| 974 | if (dma->iommu_mapped) | 989 | * mark all pages dirty if any IOMMU capable device is not able |
| | 990 | * to report dirty pages and all pages are pinned and mapped. |
| | 991 | */ |
| | 992 | if (!iommu->pinned_page_dirty_scope && dma->iommu_mapped) |
| 975 | bitmap_set(dma->bitmap, 0, nbits); | 993 | bitmap_set(dma->bitmap, 0, nbits); |
| 976 | | 994 | |
| 977 | if (shift) { | 995 | if (shift) { |
| @@ -1024,7 +1042,7 @@ static int vfio_iova_dirty_bitmap(u64 __user *bitmap, struct vfio_iommu *iommu, · 46 unmodified lines |
| 1024 | if (dma->iova > iova + size - 1) | 1042 | if (dma->iova > iova + size - 1) |
| 1025 | break; | 1043 | break; |
| 1026 | | 1044 | |
| 1027 | ret = update_user_bitmap(bitmap, dma, iova, pgsize); | 1045 | ret = update_user_bitmap(bitmap, iommu, dma, iova, pgsize); |
| 1028 | if (ret) | 1046 | if (ret) |
| 1029 | return ret; | 1047 | return ret; |
| 1030 | | 1048 | |
| @@ -1169,7 +1187,7 @@ again: · 138 unmodified lines |
| 1169 | } | 1187 | } |
| 1170 | | 1188 | |
| 1171 | if (unmap->flags & VFIO_DMA_UNMAP_FLAG_GET_DIRTY_BITMAP) { | 1189 | if (unmap->flags & VFIO_DMA_UNMAP_FLAG_GET_DIRTY_BITMAP) { |
| 1172 | ret = update_user_bitmap(bitmap->data, dma, | 1190 | ret = update_user_bitmap(bitmap->data, iommu, dma, |
| 1173 | unmap->iova, pgsize); | 1191 | unmap->iova, pgsize); |
| 1174 | if (ret) | 1192 | if (ret) |
| 1175 | break; | 1193 | break; |
| @@ -1521,6 +1539,51 @@ static struct vfio_group *find_iommu_group(struct vfio_domain *domain, · 345 unmodified lines |
| 1521 | return NULL; | 1539 | return NULL; |
| 1522 | } | 1540 | } |
| 1523 | | 1541 | |
| | 1542 | static struct vfio_group *vfio_iommu_find_iommu_group(struct vfio_iommu *iommu, |
| | 1543 | struct iommu_group *iommu_group) |
| | 1544 | { |
| | 1545 | struct vfio_domain *domain; |
| | 1546 | struct vfio_group *group = NULL; |
| | 1547 | |
| | 1548 | list_for_each_entry(domain, &iommu->domain_list, next) { |
| | 1549 | group = find_iommu_group(domain, iommu_group); |
| | 1550 | if (group) |
| | 1551 | return group; |
| | 1552 | } |
| | 1553 | |
| | 1554 | if (iommu->external_domain) |
| | 1555 | group = find_iommu_group(iommu->external_domain, iommu_group); |
| | 1556 | |
| | 1557 | return group; |
| | 1558 | } |
| | 1559 | |
| | 1560 | static void update_pinned_page_dirty_scope(struct vfio_iommu *iommu) |
| | 1561 | { |
| | 1562 | struct vfio_domain *domain; |
| | 1563 | struct vfio_group *group; |
| | 1564 | |
| | 1565 | list_for_each_entry(domain, &iommu->domain_list, next) { |
| | 1566 | list_for_each_entry(group, &domain->group_list, next) { |
| | 1567 | if (!group->pinned_page_dirty_scope) { |
| | 1568 | iommu->pinned_page_dirty_scope = false; |
| | 1569 | return; |
| | 1570 | } |
| | 1571 | } |
| | 1572 | } |
| | 1573 | |
| | 1574 | if (iommu->external_domain) { |
| | 1575 | domain = iommu->external_domain; |
| | 1576 | list_for_each_entry(group, &domain->group_list, next) { |
| | 1577 | if (!group->pinned_page_dirty_scope) { |
| | 1578 | iommu->pinned_page_dirty_scope = false; |
| | 1579 | return; |
| | 1580 | } |
| | 1581 | } |
| | 1582 | } |
| | 1583 | |
| | 1584 | iommu->pinned_page_dirty_scope = true; |
| | 1585 | } |
| | 1586 | |
| 1524 | static bool vfio_iommu_has_sw_msi(struct list_head *group_resv_regions, | 1587 | static bool vfio_iommu_has_sw_msi(struct list_head *group_resv_regions, |
| 1525 | phys_addr_t *base) | 1588 | phys_addr_t *base) |
| 1526 | { | 1589 | { |
| @@ -1928,6 +1991,16 @@ static int vfio_iommu_type1_attach_group(void *iommu_data, · 401 unmodified lines |
| 1928 | | 1991 | |
| 1929 | list_add(&group->next, | 1992 | list_add(&group->next, |
| 1930 | &iommu->external_domain->group_list); | 1993 | &iommu->external_domain->group_list); |
| | 1994 | /* |
| | 1995 | * Non-iommu backed group cannot dirty memory directly, |
| | 1996 | * it can only use interfaces that provide dirty |
| | 1997 | * tracking. |
| | 1998 | * The iommu scope can only be promoted with the |
| | 1999 | * addition of a dirty tracking group. |
| | 2000 | */ |
| | 2001 | group->pinned_page_dirty_scope = true; |
| | 2002 | if (!iommu->pinned_page_dirty_scope) |
| | 2003 | update_pinned_page_dirty_scope(iommu); |
| 1931 | mutex_unlock(&iommu->lock); | 2004 | mutex_unlock(&iommu->lock); |
| 1932 | | 2005 | |
| 1933 | return 0; | 2006 | return 0; |
| @@ -2051,6 +2124,13 @@ static int vfio_iommu_type1_attach_group(void *iommu_data, · 117 unmodified lines |
| 2051 | done: | 2124 | done: |
| 2052 | /* Delete the old one and insert new iova list */ | 2125 | /* Delete the old one and insert new iova list */ |
| 2053 | vfio_iommu_iova_insert_copy(iommu, &iova_copy); | 2126 | vfio_iommu_iova_insert_copy(iommu, &iova_copy); |
| | 2127 | |
| | 2128 | /* |
| | 2129 | * An iommu backed group can dirty memory directly and therefore |
| | 2130 | * demotes the iommu scope until it declares itself dirty tracking |
| | 2131 | * capable via the page pinning interface. |
| | 2132 | */ |
| | 2133 | iommu->pinned_page_dirty_scope = false; |
| 2054 | mutex_unlock(&iommu->lock); | 2134 | mutex_unlock(&iommu->lock); |
| 2055 | vfio_iommu_resv_free(&group_resv_regions); | 2135 | vfio_iommu_resv_free(&group_resv_regions); |
| 2056 | | 2136 | |
| @@ -2203,6 +2283,7 @@ static void vfio_iommu_type1_detach_group(void *iommu_data, · 146 unmodified lines |
| 2203 | struct vfio_iommu *iommu = iommu_data; | 2283 | struct vfio_iommu *iommu = iommu_data; |
| 2204 | struct vfio_domain *domain; | 2284 | struct vfio_domain *domain; |
| 2205 | struct vfio_group *group; | 2285 | struct vfio_group *group; |
| | 2286 | bool update_dirty_scope = false; |
| 2206 | LIST_HEAD(iova_copy); | 2287 | LIST_HEAD(iova_copy); |
| 2207 | | 2288 | |
| 2208 | mutex_lock(&iommu->lock); | 2289 | mutex_lock(&iommu->lock); |
| @@ -2210,6 +2291,7 @@ static void vfio_iommu_type1_detach_group(void *iommu_data, · 1 unmodified line |
| 2210 | if (iommu->external_domain) { | 2291 | if (iommu->external_domain) { |
| 2211 | group = find_iommu_group(iommu->external_domain, iommu_group); | 2292 | group = find_iommu_group(iommu->external_domain, iommu_group); |
| 2212 | if (group) { | 2293 | if (group) { |
| | 2294 | update_dirty_scope = !group->pinned_page_dirty_scope; |
| 2213 | list_del(&group->next); | 2295 | list_del(&group->next); |
| 2214 | kfree(group); | 2296 | kfree(group); |
| 2215 | | 2297 | |
| @@ -2239,6 +2321,7 @@ static void vfio_iommu_type1_detach_group(void *iommu_data, · 23 unmodified lines |
| 2239 | continue; | 2321 | continue; |
| 2240 | | 2322 | |
| 2241 | vfio_iommu_detach_group(domain, group); | 2323 | vfio_iommu_detach_group(domain, group); |
| | 2324 | update_dirty_scope = !group->pinned_page_dirty_scope; |
| 2242 | list_del(&group->next); | 2325 | list_del(&group->next); |
| 2243 | kfree(group); | 2326 | kfree(group); |
| 2244 | /* | 2327 | /* |
| @@ -2270,6 +2353,12 @@ static void vfio_iommu_type1_detach_group(void *iommu_data, · 25 unmodified lines |
| 2270 | vfio_iommu_iova_free(&iova_copy); | 2353 | vfio_iommu_iova_free(&iova_copy); |
| 2271 | | 2354 | |
| 2272 | detach_group_done: | 2355 | detach_group_done: |
| | 2356 | /* |
| | 2357 | * Removal of a group without dirty tracking may allow the iommu scope |
| | 2358 | * to be promoted. |
| | 2359 | */ |
| | 2360 | if (update_dirty_scope) |
| | 2361 | update_pinned_page_dirty_scope(iommu); |
| 2273 | mutex_unlock(&iommu->lock); | 2362 | mutex_unlock(&iommu->lock); |
| 2274 | } | 2363 | } |
| 2275 | | 2364 | |