History

vfio: Selective dirty page tracking if IOMMU backed device pins pages

Added a check such that only singleton IOMMU groups can pin pages. >From the point when vendor driver pins any pages, consider IOMMU group dirty page scope to be limited to pinned pages. To optimize to avoid walking list often, added flag pinned_page_dirty_scope to indicate if all of the vfio_groups for each vfio_domain in the domain_list dirty page scope is limited to pinned pages. This flag is updated on first pinned pages request for that IOMMU group and on attaching/detaching group.

Kirti Wankhede committed 6 years ago

Commit 95fc87b1 parent ad721703 files · +109 −11Browse files

Unified · Split

3 files
  1. drivers/vfio/vfio.c +10 −3
  2. drivers/vfio/vfio_iommu_type1.c +96 −7
  3. include/linux/vfio.h +3 −1
drivers/vfio/vfio.c+10−3
@@ -85,6 +85,7 @@ struct vfio_group { · 84 unmodified lines
85 atomic_t opened;85 atomic_t opened;
86 wait_queue_head_t container_q;86 wait_queue_head_t container_q;
87 bool noiommu;87 bool noiommu;
88 unsigned int dev_counter;
88 struct kvm *kvm;89 struct kvm *kvm;
89 struct blocking_notifier_head notifier;90 struct blocking_notifier_head notifier;
90};91};
@@ -555,6 +556,7 @@ struct vfio_device *vfio_group_create_device(struct vfio_group *group, · 464 unmodified lines
555 556
556 mutex_lock(&group->device_lock);557 mutex_lock(&group->device_lock);
557 list_add(&device->group_next, &group->device_list);558 list_add(&device->group_next, &group->device_list);
559 group->dev_counter++;
558 mutex_unlock(&group->device_lock);560 mutex_unlock(&group->device_lock);
559 561
560 return device;562 return device;
@@ -567,6 +569,7 @@ static void vfio_device_release(struct kref *kref) · 6 unmodified lines
567 struct vfio_group *group = device->group;569 struct vfio_group *group = device->group;
568 570
569 list_del(&device->group_next);571 list_del(&device->group_next);
572 group->dev_counter--;
570 mutex_unlock(&group->device_lock);573 mutex_unlock(&group->device_lock);
571 574
572 dev_set_drvdata(device->dev, NULL);575 dev_set_drvdata(device->dev, NULL);
@@ -1945,6 +1948,9 @@ int vfio_pin_pages(struct device *dev, unsigned long *user_pfn, int npage, · 1372 unmodified lines
1945 if (!group)1948 if (!group)
1946 return -ENODEV;1949 return -ENODEV;
1947 1950
1951 if (group->dev_counter > 1)
1952 return -EINVAL;
1953
1948 ret = vfio_group_add_container_user(group);1954 ret = vfio_group_add_container_user(group);
1949 if (ret)1955 if (ret)
1950 goto err_pin_pages;1956 goto err_pin_pages;
@@ -1952,7 +1958,8 @@ int vfio_pin_pages(struct device *dev, unsigned long *user_pfn, int npage, · 1 unmodified line
1952 container = group->container;1958 container = group->container;
1953 driver = container->iommu_driver;1959 driver = container->iommu_driver;
1954 if (likely(driver && driver->ops->pin_pages))1960 if (likely(driver && driver->ops->pin_pages))
1955 ret = driver->ops->pin_pages(container->iommu_data, user_pfn,1961 ret = driver->ops->pin_pages(container->iommu_data,
1962 group->iommu_group, user_pfn,
1956 npage, prot, phys_pfn);1963 npage, prot, phys_pfn);
1957 else1964 else
1958 ret = -ENOTTY;1965 ret = -ENOTTY;
@@ -2050,8 +2057,8 @@ int vfio_group_pin_pages(struct vfio_group *group, · 91 unmodified lines
2050 driver = container->iommu_driver;2057 driver = container->iommu_driver;
2051 if (likely(driver && driver->ops->pin_pages))2058 if (likely(driver && driver->ops->pin_pages))
2052 ret = driver->ops->pin_pages(container->iommu_data,2059 ret = driver->ops->pin_pages(container->iommu_data,
2053 user_iova_pfn, npage,2060 group->iommu_group, user_iova_pfn,
2054 prot, phys_pfn);2061 npage, prot, phys_pfn);
2055 else2062 else
2056 ret = -ENOTTY;2063 ret = -ENOTTY;
2057 2064
drivers/vfio/vfio_iommu_type1.c+96−7
@@ -73,6 +73,7 @@ struct vfio_iommu { · 72 unmodified lines
73 bool v2;73 bool v2;
74 bool nesting;74 bool nesting;
75 bool dirty_page_tracking;75 bool dirty_page_tracking;
76 bool pinned_page_dirty_scope;
76};77};
77 78
78struct vfio_domain {79struct vfio_domain {
@@ -100,6 +101,7 @@ struct vfio_group { · 21 unmodified lines
100 struct iommu_group *iommu_group;101 struct iommu_group *iommu_group;
101 struct list_head next;102 struct list_head next;
102 bool mdev_group; /* An mdev group */103 bool mdev_group; /* An mdev group */
104 bool pinned_page_dirty_scope;
103};105};
104 106
105struct vfio_iova {107struct vfio_iova {
@@ -143,6 +145,10 @@ struct vfio_regions { · 37 unmodified lines
143 145
144static int put_pfn(unsigned long pfn, int prot);146static int put_pfn(unsigned long pfn, int prot);
145 147
148static struct vfio_group *vfio_iommu_find_iommu_group(struct vfio_iommu *iommu,
149 struct iommu_group *iommu_group);
150
151static void update_pinned_page_dirty_scope(struct vfio_iommu *iommu);
146/*152/*
147 * This code handles mapping and unmapping of user data buffers153 * This code handles mapping and unmapping of user data buffers
148 * into DMA'ble space using the IOMMU154 * into DMA'ble space using the IOMMU
@@ -622,11 +628,13 @@ static int vfio_unpin_page_external(struct vfio_dma *dma, dma_addr_t iova, · 473 unmodified lines
622}628}
623 629
624static int vfio_iommu_type1_pin_pages(void *iommu_data,630static int vfio_iommu_type1_pin_pages(void *iommu_data,
631 struct iommu_group *iommu_group,
625 unsigned long *user_pfn,632 unsigned long *user_pfn,
626 int npage, int prot,633 int npage, int prot,
627 unsigned long *phys_pfn)634 unsigned long *phys_pfn)
628{635{
629 struct vfio_iommu *iommu = iommu_data;636 struct vfio_iommu *iommu = iommu_data;
637 struct vfio_group *group;
630 int i, j, ret;638 int i, j, ret;
631 unsigned long remote_vaddr;639 unsigned long remote_vaddr;
632 struct vfio_dma *dma;640 struct vfio_dma *dma;
@@ -699,8 +707,14 @@ static int vfio_iommu_type1_pin_pages(void *iommu_data, · 66 unmodified lines
699 (iova - dma->iova) >> pgshift, 1);707 (iova - dma->iova) >> pgshift, 1);
700 }708 }
701 }709 }
702
703 ret = i;710 ret = i;
711
712 group = vfio_iommu_find_iommu_group(iommu, iommu_group);
713 if (!group->pinned_page_dirty_scope) {
714 group->pinned_page_dirty_scope = true;
715 update_pinned_page_dirty_scope(iommu);
716 }
717
704 goto pin_done;718 goto pin_done;
705 719
706pin_unwind:720pin_unwind:
@@ -960,8 +974,9 @@ static void vfio_update_pgsize_bitmap(struct vfio_iommu *iommu) · 253 unmodified lines
960 }974 }
961}975}
962 976
963static int update_user_bitmap(u64 __user *bitmap, struct vfio_dma *dma,977static int update_user_bitmap(u64 __user *bitmap, struct vfio_iommu *iommu,
964 dma_addr_t base_iova, size_t pgsize)978 struct vfio_dma *dma, dma_addr_t base_iova,
979 size_t pgsize)
965{980{
966 unsigned long pgshift = __ffs(pgsize);981 unsigned long pgshift = __ffs(pgsize);
967 unsigned long nbits = dma->size >> pgshift;982 unsigned long nbits = dma->size >> pgshift;
@@ -970,8 +985,11 @@ static int update_user_bitmap(u64 __user *bitmap, struct vfio_dma *dma, · 2 unmodified lines
970 unsigned long shift = bit_offset % BITS_PER_LONG;985 unsigned long shift = bit_offset % BITS_PER_LONG;
971 unsigned long leftover;986 unsigned long leftover;
972 987
973 /* mark all pages dirty if all pages are pinned and mapped. */988 /*
974 if (dma->iommu_mapped)989 * mark all pages dirty if any IOMMU capable device is not able
990 * to report dirty pages and all pages are pinned and mapped.
991 */
992 if (!iommu->pinned_page_dirty_scope && dma->iommu_mapped)
975 bitmap_set(dma->bitmap, 0, nbits);993 bitmap_set(dma->bitmap, 0, nbits);
976 994
977 if (shift) {995 if (shift) {
@@ -1024,7 +1042,7 @@ static int vfio_iova_dirty_bitmap(u64 __user *bitmap, struct vfio_iommu *iommu, · 46 unmodified lines
1024 if (dma->iova > iova + size - 1)1042 if (dma->iova > iova + size - 1)
1025 break;1043 break;
1026 1044
1027 ret = update_user_bitmap(bitmap, dma, iova, pgsize);1045 ret = update_user_bitmap(bitmap, iommu, dma, iova, pgsize);
1028 if (ret)1046 if (ret)
1029 return ret;1047 return ret;
1030 1048
@@ -1169,7 +1187,7 @@ again: · 138 unmodified lines
1169 }1187 }
1170 1188
1171 if (unmap->flags & VFIO_DMA_UNMAP_FLAG_GET_DIRTY_BITMAP) {1189 if (unmap->flags & VFIO_DMA_UNMAP_FLAG_GET_DIRTY_BITMAP) {
1172 ret = update_user_bitmap(bitmap->data, dma,1190 ret = update_user_bitmap(bitmap->data, iommu, dma,
1173 unmap->iova, pgsize);1191 unmap->iova, pgsize);
1174 if (ret)1192 if (ret)
1175 break;1193 break;
@@ -1521,6 +1539,51 @@ static struct vfio_group *find_iommu_group(struct vfio_domain *domain, · 345 unmodified lines
1521 return NULL;1539 return NULL;
1522}1540}
1523 1541
1542static struct vfio_group *vfio_iommu_find_iommu_group(struct vfio_iommu *iommu,
1543 struct iommu_group *iommu_group)
1544{
1545 struct vfio_domain *domain;
1546 struct vfio_group *group = NULL;
1547
1548 list_for_each_entry(domain, &iommu->domain_list, next) {
1549 group = find_iommu_group(domain, iommu_group);
1550 if (group)
1551 return group;
1552 }
1553
1554 if (iommu->external_domain)
1555 group = find_iommu_group(iommu->external_domain, iommu_group);
1556
1557 return group;
1558}
1559
1560static void update_pinned_page_dirty_scope(struct vfio_iommu *iommu)
1561{
1562 struct vfio_domain *domain;
1563 struct vfio_group *group;
1564
1565 list_for_each_entry(domain, &iommu->domain_list, next) {
1566 list_for_each_entry(group, &domain->group_list, next) {
1567 if (!group->pinned_page_dirty_scope) {
1568 iommu->pinned_page_dirty_scope = false;
1569 return;
1570 }
1571 }
1572 }
1573
1574 if (iommu->external_domain) {
1575 domain = iommu->external_domain;
1576 list_for_each_entry(group, &domain->group_list, next) {
1577 if (!group->pinned_page_dirty_scope) {
1578 iommu->pinned_page_dirty_scope = false;
1579 return;
1580 }
1581 }
1582 }
1583
1584 iommu->pinned_page_dirty_scope = true;
1585}
1586
1524static bool vfio_iommu_has_sw_msi(struct list_head *group_resv_regions,1587static bool vfio_iommu_has_sw_msi(struct list_head *group_resv_regions,
1525 phys_addr_t *base)1588 phys_addr_t *base)
1526{1589{
@@ -1928,6 +1991,16 @@ static int vfio_iommu_type1_attach_group(void *iommu_data, · 401 unmodified lines
1928 1991
1929 list_add(&group->next,1992 list_add(&group->next,
1930 &iommu->external_domain->group_list);1993 &iommu->external_domain->group_list);
1994 /*
1995 * Non-iommu backed group cannot dirty memory directly,
1996 * it can only use interfaces that provide dirty
1997 * tracking.
1998 * The iommu scope can only be promoted with the
1999 * addition of a dirty tracking group.
2000 */
2001 group->pinned_page_dirty_scope = true;
2002 if (!iommu->pinned_page_dirty_scope)
2003 update_pinned_page_dirty_scope(iommu);
1931 mutex_unlock(&iommu->lock);2004 mutex_unlock(&iommu->lock);
1932 2005
1933 return 0;2006 return 0;
@@ -2051,6 +2124,13 @@ static int vfio_iommu_type1_attach_group(void *iommu_data, · 117 unmodified lines
2051done:2124done:
2052 /* Delete the old one and insert new iova list */2125 /* Delete the old one and insert new iova list */
2053 vfio_iommu_iova_insert_copy(iommu, &iova_copy);2126 vfio_iommu_iova_insert_copy(iommu, &iova_copy);
2127
2128 /*
2129 * An iommu backed group can dirty memory directly and therefore
2130 * demotes the iommu scope until it declares itself dirty tracking
2131 * capable via the page pinning interface.
2132 */
2133 iommu->pinned_page_dirty_scope = false;
2054 mutex_unlock(&iommu->lock);2134 mutex_unlock(&iommu->lock);
2055 vfio_iommu_resv_free(&group_resv_regions);2135 vfio_iommu_resv_free(&group_resv_regions);
2056 2136
@@ -2203,6 +2283,7 @@ static void vfio_iommu_type1_detach_group(void *iommu_data, · 146 unmodified lines
2203 struct vfio_iommu *iommu = iommu_data;2283 struct vfio_iommu *iommu = iommu_data;
2204 struct vfio_domain *domain;2284 struct vfio_domain *domain;
2205 struct vfio_group *group;2285 struct vfio_group *group;
2286 bool update_dirty_scope = false;
2206 LIST_HEAD(iova_copy);2287 LIST_HEAD(iova_copy);
2207 2288
2208 mutex_lock(&iommu->lock);2289 mutex_lock(&iommu->lock);
@@ -2210,6 +2291,7 @@ static void vfio_iommu_type1_detach_group(void *iommu_data, · 1 unmodified line
2210 if (iommu->external_domain) {2291 if (iommu->external_domain) {
2211 group = find_iommu_group(iommu->external_domain, iommu_group);2292 group = find_iommu_group(iommu->external_domain, iommu_group);
2212 if (group) {2293 if (group) {
2294 update_dirty_scope = !group->pinned_page_dirty_scope;
2213 list_del(&group->next);2295 list_del(&group->next);
2214 kfree(group);2296 kfree(group);
2215 2297
@@ -2239,6 +2321,7 @@ static void vfio_iommu_type1_detach_group(void *iommu_data, · 23 unmodified lines
2239 continue;2321 continue;
2240 2322
2241 vfio_iommu_detach_group(domain, group);2323 vfio_iommu_detach_group(domain, group);
2324 update_dirty_scope = !group->pinned_page_dirty_scope;
2242 list_del(&group->next);2325 list_del(&group->next);
2243 kfree(group);2326 kfree(group);
2244 /*2327 /*
@@ -2270,6 +2353,12 @@ static void vfio_iommu_type1_detach_group(void *iommu_data, · 25 unmodified lines
2270 vfio_iommu_iova_free(&iova_copy);2353 vfio_iommu_iova_free(&iova_copy);
2271 2354
2272detach_group_done:2355detach_group_done:
2356 /*
2357 * Removal of a group without dirty tracking may allow the iommu scope
2358 * to be promoted.
2359 */
2360 if (update_dirty_scope)
2361 update_pinned_page_dirty_scope(iommu);
2273 mutex_unlock(&iommu->lock);2362 mutex_unlock(&iommu->lock);
2274}2363}
2275 2364
include/linux/vfio.h+3−1
@@ -76,7 +76,9 @@ struct vfio_iommu_driver_ops { · 75 unmodified lines
76 struct iommu_group *group);76 struct iommu_group *group);
77 void (*detach_group)(void *iommu_data,77 void (*detach_group)(void *iommu_data,
78 struct iommu_group *group);78 struct iommu_group *group);
79 int (*pin_pages)(void *iommu_data, unsigned long *user_pfn,79 int (*pin_pages)(void *iommu_data,
80 struct iommu_group *group,
81 unsigned long *user_pfn,
80 int npage, int prot,82 int npage, int prot,
81 unsigned long *phys_pfn);83 unsigned long *phys_pfn);
82 int (*unpin_pages)(void *iommu_data,84 int (*unpin_pages)(void *iommu_data,

Keyboard shortcuts

General

Show keyboard shortcuts
?
Focus the current search
/

Issues and pull requests

Open filters
f

Issues, pull requests and Inbox

Select next item
j↓
Select previous item
k↑
Open selected item
Enter
Dismiss filters or clear selection
Esc

Repository code

Go to file
t