History

vfio: Selective dirty page tracking if IOMMU backed device pins pages

Added a check such that only singleton IOMMU groups can pin pages. >From the point when vendor driver pins any pages, consider IOMMU group dirty page scope to be limited to pinned pages. To optimize to avoid walking list often, added flag pinned_page_dirty_scope to indicate if all of the vfio_groups for each vfio_domain in the domain_list dirty page scope is limited to pinned pages. This flag is updated on first pinned pages request for that IOMMU group and on attaching/detaching group.

Kirti Wankhede committed 6 years ago

Commit 95fc87b1 parent ad721703 files · +109 −11Browse files

Unified · Split

3 files
  1. drivers/vfio/vfio.c +10 −3
  2. drivers/vfio/vfio_iommu_type1.c +96 −7
  3. include/linux/vfio.h +3 −1
drivers/vfio/vfio.c+10−3
@@ -85,6 +85,7 @@ struct vfio_group { · 84 unmodified lines
8585 atomic_t opened;
8686 wait_queue_head_t container_q;
8787 bool noiommu;
88+ unsigned int dev_counter;
8889 struct kvm *kvm;
8990 struct blocking_notifier_head notifier;
9091 };
@@ -555,6 +556,7 @@ struct vfio_device *vfio_group_create_device(struct vfio_group *group, · 464 unmodified lines
555556
556557 mutex_lock(&group->device_lock);
557558 list_add(&device->group_next, &group->device_list);
559+ group->dev_counter++;
558560 mutex_unlock(&group->device_lock);
559561
560562 return device;
@@ -567,6 +569,7 @@ static void vfio_device_release(struct kref *kref) · 6 unmodified lines
567569 struct vfio_group *group = device->group;
568570
569571 list_del(&device->group_next);
572+ group->dev_counter--;
570573 mutex_unlock(&group->device_lock);
571574
572575 dev_set_drvdata(device->dev, NULL);
@@ -1945,6 +1948,9 @@ int vfio_pin_pages(struct device *dev, unsigned long *user_pfn, int npage, · 1372 unmodified lines
19451948 if (!group)
19461949 return -ENODEV;
19471950
1951+ if (group->dev_counter > 1)
1952+ return -EINVAL;
1953+
19481954 ret = vfio_group_add_container_user(group);
19491955 if (ret)
19501956 goto err_pin_pages;
@@ -1952,7 +1958,8 @@ int vfio_pin_pages(struct device *dev, unsigned long *user_pfn, int npage, · 1 unmodified line
19521958 container = group->container;
19531959 driver = container->iommu_driver;
19541960 if (likely(driver && driver->ops->pin_pages))
1955− ret = driver->ops->pin_pages(container->iommu_data, user_pfn,
1961+ ret = driver->ops->pin_pages(container->iommu_data,
1962+ group->iommu_group, user_pfn,
19561963 npage, prot, phys_pfn);
19571964 else
19581965 ret = -ENOTTY;
@@ -2050,8 +2057,8 @@ int vfio_group_pin_pages(struct vfio_group *group, · 91 unmodified lines
20502057 driver = container->iommu_driver;
20512058 if (likely(driver && driver->ops->pin_pages))
20522059 ret = driver->ops->pin_pages(container->iommu_data,
2053− user_iova_pfn, npage,
2054− prot, phys_pfn);
2060+ group->iommu_group, user_iova_pfn,
2061+ npage, prot, phys_pfn);
20552062 else
20562063 ret = -ENOTTY;
20572064
drivers/vfio/vfio_iommu_type1.c+96−7
@@ -73,6 +73,7 @@ struct vfio_iommu { · 72 unmodified lines
7373 bool v2;
7474 bool nesting;
7575 bool dirty_page_tracking;
76+ bool pinned_page_dirty_scope;
7677 };
7778
7879 struct vfio_domain {
@@ -100,6 +101,7 @@ struct vfio_group { · 21 unmodified lines
100101 struct iommu_group *iommu_group;
101102 struct list_head next;
102103 bool mdev_group; /* An mdev group */
104+ bool pinned_page_dirty_scope;
103105 };
104106
105107 struct vfio_iova {
@@ -143,6 +145,10 @@ struct vfio_regions { · 37 unmodified lines
143145
144146 static int put_pfn(unsigned long pfn, int prot);
145147
148+static struct vfio_group *vfio_iommu_find_iommu_group(struct vfio_iommu *iommu,
149+ struct iommu_group *iommu_group);
150+
151+static void update_pinned_page_dirty_scope(struct vfio_iommu *iommu);
146152 /*
147153 * This code handles mapping and unmapping of user data buffers
148154 * into DMA'ble space using the IOMMU
@@ -622,11 +628,13 @@ static int vfio_unpin_page_external(struct vfio_dma *dma, dma_addr_t iova, · 473 unmodified lines
622628 }
623629
624630 static int vfio_iommu_type1_pin_pages(void *iommu_data,
631+ struct iommu_group *iommu_group,
625632 unsigned long *user_pfn,
626633 int npage, int prot,
627634 unsigned long *phys_pfn)
628635 {
629636 struct vfio_iommu *iommu = iommu_data;
637+ struct vfio_group *group;
630638 int i, j, ret;
631639 unsigned long remote_vaddr;
632640 struct vfio_dma *dma;
@@ -699,8 +707,14 @@ static int vfio_iommu_type1_pin_pages(void *iommu_data, · 66 unmodified lines
699707 (iova - dma->iova) >> pgshift, 1);
700708 }
701709 }
702−
703710 ret = i;
711+
712+ group = vfio_iommu_find_iommu_group(iommu, iommu_group);
713+ if (!group->pinned_page_dirty_scope) {
714+ group->pinned_page_dirty_scope = true;
715+ update_pinned_page_dirty_scope(iommu);
716+ }
717+
704718 goto pin_done;
705719
706720 pin_unwind:
@@ -960,8 +974,9 @@ static void vfio_update_pgsize_bitmap(struct vfio_iommu *iommu) · 253 unmodified lines
960974 }
961975 }
962976
963−static int update_user_bitmap(u64 __user *bitmap, struct vfio_dma *dma,
964− dma_addr_t base_iova, size_t pgsize)
977+static int update_user_bitmap(u64 __user *bitmap, struct vfio_iommu *iommu,
978+ struct vfio_dma *dma, dma_addr_t base_iova,
979+ size_t pgsize)
965980 {
966981 unsigned long pgshift = __ffs(pgsize);
967982 unsigned long nbits = dma->size >> pgshift;
@@ -970,8 +985,11 @@ static int update_user_bitmap(u64 __user *bitmap, struct vfio_dma *dma, · 2 unmodified lines
970985 unsigned long shift = bit_offset % BITS_PER_LONG;
971986 unsigned long leftover;
972987
973− /* mark all pages dirty if all pages are pinned and mapped. */
974− if (dma->iommu_mapped)
988+ /*
989+ * mark all pages dirty if any IOMMU capable device is not able
990+ * to report dirty pages and all pages are pinned and mapped.
991+ */
992+ if (!iommu->pinned_page_dirty_scope && dma->iommu_mapped)
975993 bitmap_set(dma->bitmap, 0, nbits);
976994
977995 if (shift) {
@@ -1024,7 +1042,7 @@ static int vfio_iova_dirty_bitmap(u64 __user *bitmap, struct vfio_iommu *iommu, · 46 unmodified lines
10241042 if (dma->iova > iova + size - 1)
10251043 break;
10261044
1027− ret = update_user_bitmap(bitmap, dma, iova, pgsize);
1045+ ret = update_user_bitmap(bitmap, iommu, dma, iova, pgsize);
10281046 if (ret)
10291047 return ret;
10301048
@@ -1169,7 +1187,7 @@ again: · 138 unmodified lines
11691187 }
11701188
11711189 if (unmap->flags & VFIO_DMA_UNMAP_FLAG_GET_DIRTY_BITMAP) {
1172− ret = update_user_bitmap(bitmap->data, dma,
1190+ ret = update_user_bitmap(bitmap->data, iommu, dma,
11731191 unmap->iova, pgsize);
11741192 if (ret)
11751193 break;
@@ -1521,6 +1539,51 @@ static struct vfio_group *find_iommu_group(struct vfio_domain *domain, · 345 unmodified lines
15211539 return NULL;
15221540 }
15231541
1542+static struct vfio_group *vfio_iommu_find_iommu_group(struct vfio_iommu *iommu,
1543+ struct iommu_group *iommu_group)
1544+{
1545+ struct vfio_domain *domain;
1546+ struct vfio_group *group = NULL;
1547+
1548+ list_for_each_entry(domain, &iommu->domain_list, next) {
1549+ group = find_iommu_group(domain, iommu_group);
1550+ if (group)
1551+ return group;
1552+ }
1553+
1554+ if (iommu->external_domain)
1555+ group = find_iommu_group(iommu->external_domain, iommu_group);
1556+
1557+ return group;
1558+}
1559+
1560+static void update_pinned_page_dirty_scope(struct vfio_iommu *iommu)
1561+{
1562+ struct vfio_domain *domain;
1563+ struct vfio_group *group;
1564+
1565+ list_for_each_entry(domain, &iommu->domain_list, next) {
1566+ list_for_each_entry(group, &domain->group_list, next) {
1567+ if (!group->pinned_page_dirty_scope) {
1568+ iommu->pinned_page_dirty_scope = false;
1569+ return;
1570+ }
1571+ }
1572+ }
1573+
1574+ if (iommu->external_domain) {
1575+ domain = iommu->external_domain;
1576+ list_for_each_entry(group, &domain->group_list, next) {
1577+ if (!group->pinned_page_dirty_scope) {
1578+ iommu->pinned_page_dirty_scope = false;
1579+ return;
1580+ }
1581+ }
1582+ }
1583+
1584+ iommu->pinned_page_dirty_scope = true;
1585+}
1586+
15241587 static bool vfio_iommu_has_sw_msi(struct list_head *group_resv_regions,
15251588 phys_addr_t *base)
15261589 {
@@ -1928,6 +1991,16 @@ static int vfio_iommu_type1_attach_group(void *iommu_data, · 401 unmodified lines
19281991
19291992 list_add(&group->next,
19301993 &iommu->external_domain->group_list);
1994+ /*
1995+ * Non-iommu backed group cannot dirty memory directly,
1996+ * it can only use interfaces that provide dirty
1997+ * tracking.
1998+ * The iommu scope can only be promoted with the
1999+ * addition of a dirty tracking group.
2000+ */
2001+ group->pinned_page_dirty_scope = true;
2002+ if (!iommu->pinned_page_dirty_scope)
2003+ update_pinned_page_dirty_scope(iommu);
19312004 mutex_unlock(&iommu->lock);
19322005
19332006 return 0;
@@ -2051,6 +2124,13 @@ static int vfio_iommu_type1_attach_group(void *iommu_data, · 117 unmodified lines
20512124 done:
20522125 /* Delete the old one and insert new iova list */
20532126 vfio_iommu_iova_insert_copy(iommu, &iova_copy);
2127+
2128+ /*
2129+ * An iommu backed group can dirty memory directly and therefore
2130+ * demotes the iommu scope until it declares itself dirty tracking
2131+ * capable via the page pinning interface.
2132+ */
2133+ iommu->pinned_page_dirty_scope = false;
20542134 mutex_unlock(&iommu->lock);
20552135 vfio_iommu_resv_free(&group_resv_regions);
20562136
@@ -2203,6 +2283,7 @@ static void vfio_iommu_type1_detach_group(void *iommu_data, · 146 unmodified lines
22032283 struct vfio_iommu *iommu = iommu_data;
22042284 struct vfio_domain *domain;
22052285 struct vfio_group *group;
2286+ bool update_dirty_scope = false;
22062287 LIST_HEAD(iova_copy);
22072288
22082289 mutex_lock(&iommu->lock);
@@ -2210,6 +2291,7 @@ static void vfio_iommu_type1_detach_group(void *iommu_data, · 1 unmodified line
22102291 if (iommu->external_domain) {
22112292 group = find_iommu_group(iommu->external_domain, iommu_group);
22122293 if (group) {
2294+ update_dirty_scope = !group->pinned_page_dirty_scope;
22132295 list_del(&group->next);
22142296 kfree(group);
22152297
@@ -2239,6 +2321,7 @@ static void vfio_iommu_type1_detach_group(void *iommu_data, · 23 unmodified lines
22392321 continue;
22402322
22412323 vfio_iommu_detach_group(domain, group);
2324+ update_dirty_scope = !group->pinned_page_dirty_scope;
22422325 list_del(&group->next);
22432326 kfree(group);
22442327 /*
@@ -2270,6 +2353,12 @@ static void vfio_iommu_type1_detach_group(void *iommu_data, · 25 unmodified lines
22702353 vfio_iommu_iova_free(&iova_copy);
22712354
22722355 detach_group_done:
2356+ /*
2357+ * Removal of a group without dirty tracking may allow the iommu scope
2358+ * to be promoted.
2359+ */
2360+ if (update_dirty_scope)
2361+ update_pinned_page_dirty_scope(iommu);
22732362 mutex_unlock(&iommu->lock);
22742363 }
22752364
include/linux/vfio.h+3−1
@@ -76,7 +76,9 @@ struct vfio_iommu_driver_ops { · 75 unmodified lines
7676 struct iommu_group *group);
7777 void (*detach_group)(void *iommu_data,
7878 struct iommu_group *group);
79− int (*pin_pages)(void *iommu_data, unsigned long *user_pfn,
79+ int (*pin_pages)(void *iommu_data,
80+ struct iommu_group *group,
81+ unsigned long *user_pfn,
8082 int npage, int prot,
8183 unsigned long *phys_pfn);
8284 int (*unpin_pages)(void *iommu_data,

Keyboard shortcuts

General

Show keyboard shortcuts
?
Focus the current search
/

Issues and pull requests

Open filters
f

Issues, pull requests and Inbox

Select next item
j↓
Select previous item
k↑
Open selected item
Enter
Dismiss filters or clear selection
Esc

Repository code

Go to file
t