History

vfio: hugepage support for vfio_iommu_type1

We currently send all mappings to the iommu in PAGE_SIZE chunks, which prevents the iommu from enabling support for larger page sizes. We still need to pin pages, which means we step through them in PAGE_SIZE chunks, but we can batch up contiguous physical memory chunks to allow the iommu the opportunity to use larger pages. The approach here is a bit different that the one currently used for legacy KVM device assignment. Rather than looking at the vma page size and using that as the maximum size to pass to the iommu, we instead simply look at whether the next page is physically contiguous. This means we might ask the iommu to map a 4MB region, while legacy KVM might limit itself to a maximum of 2MB. Splitting our mapping path also allows us to be smarter about locked memory because we can more easily unwind if the user attempts to exceed the limit. Therefore, rather than assuming that a mapping will result in locked memory, we test each page as it is pinned to determine whether it locks RAM vs an mmap'd MMIO region. This should result in better locking granularity and less locked page fudge factors in userspace. The unmap path uses the same algorithm as legacy KVM. We don't want to track the pfn for each mapping ourselves, but we need the pfn in order to unpin pages. We therefore ask the iommu for the iova to physical address translation, ask it to unpin a page, and see how many pages were actually unpinned. iommus supporting large pages will often return something bigger than a page here, which we know will be physically contiguous and we can unpin a batch of pfns. iommus not supporting large mappings won't see an improvement in batching here as they only unmap a page at a time. With this change, we also make a clarification to the API for mapping and unmapping DMA. We can only guarantee unmaps at the same granularity as used for the original mapping. In other words, unmapping a subregion of a previous mapping is not guaranteed and may result in a larger or smaller unmapping than requested. The size field in the unmapping structure is updated to reflect this. Previously this was unmodified on mapping, always returning the the requested unmap size. This is now updated to return the actual unmap size on success, allowing userspace to appropriately track mappings.

Alex Williamson committed 13 years ago

Commit 166fd7d1 parent cd9b2262 files · +344 −187Browse files

Unified · Split

2 files
  1. drivers/vfio/vfio_iommu_type1.c +338 −185
  2. include/uapi/linux/vfio.h +6 −2
drivers/vfio/vfio_iommu_type1.c+338−185
@@ -60,7 +60,7 @@ struct vfio_dma { · 59 unmodified lines
6060 struct rb_node node;
6161 dma_addr_t iova; /* Device address */
6262 unsigned long vaddr; /* Process virtual addr */
63− long npage; /* Number of pages */
63+ size_t size; /* Map size (bytes) */
6464 int prot; /* IOMMU_READ/WRITE */
6565 };
6666
@@ -74,8 +74,6 @@ struct vfio_group { · 7 unmodified lines
7474 * into DMA'ble space using the IOMMU
7575 */
7676
77−#define NPAGE_TO_SIZE(npage) ((size_t)(npage) << PAGE_SHIFT)
78−
7977 static struct vfio_dma *vfio_find_dma(struct vfio_iommu *iommu,
8078 dma_addr_t start, size_t size)
8179 {
@@ -86,7 +84,7 @@ static struct vfio_dma *vfio_find_dma(struct vfio_iommu *iommu, · 4 unmodified lines
8684
8785 if (start + size <= dma->iova)
8886 node = node->rb_left;
89− else if (start >= dma->iova + NPAGE_TO_SIZE(dma->npage))
87+ else if (start >= dma->iova + dma->size)
9088 node = node->rb_right;
9189 else
9290 return dma;
@@ -104,7 +102,7 @@ static void vfio_insert_dma(struct vfio_iommu *iommu, struct vfio_dma *new) · 11 unmodified lines
104102 parent = *link;
105103 dma = rb_entry(parent, struct vfio_dma, node);
106104
107− if (new->iova + NPAGE_TO_SIZE(new->npage) <= dma->iova)
105+ if (new->iova + new->size <= dma->iova)
108106 link = &(*link)->rb_left;
109107 else
110108 link = &(*link)->rb_right;
@@ -144,8 +142,8 @@ static void vfio_lock_acct(long npage) · 33 unmodified lines
144142 struct vwork *vwork;
145143 struct mm_struct *mm;
146144
147− if (!current->mm)
148− return; /* process exited */
145+ if (!current->mm || !npage)
146+ return; /* process exited or nothing to do */
149147
150148 if (down_write_trylock(&current->mm->mmap_sem)) {
151149 current->mm->locked_vm += npage;
@@ -217,33 +215,6 @@ static int put_pfn(unsigned long pfn, int prot) · 65 unmodified lines
217215 return 0;
218216 }
219217
220−/* Unmap DMA region */
221−static long __vfio_dma_do_unmap(struct vfio_iommu *iommu, dma_addr_t iova,
222− long npage, int prot)
223−{
224− long i, unlocked = 0;
225−
226− for (i = 0; i < npage; i++, iova += PAGE_SIZE) {
227− unsigned long pfn;
228−
229− pfn = iommu_iova_to_phys(iommu->domain, iova) >> PAGE_SHIFT;
230− if (pfn) {
231− iommu_unmap(iommu->domain, iova, PAGE_SIZE);
232− unlocked += put_pfn(pfn, prot);
233− }
234− }
235− return unlocked;
236−}
237−
238−static void vfio_dma_unmap(struct vfio_iommu *iommu, dma_addr_t iova,
239− long npage, int prot)
240−{
241− long unlocked;
242−
243− unlocked = __vfio_dma_do_unmap(iommu, iova, npage, prot);
244− vfio_lock_acct(-unlocked);
245−}
246−
247218 static int vaddr_get_pfn(unsigned long vaddr, int prot, unsigned long *pfn)
248219 {
249220 struct page *page[1];
@@ -270,79 +241,142 @@ static int vaddr_get_pfn(unsigned long vaddr, int prot, unsigned long *pfn) · 20 unmodified lines
270241 return ret;
271242 }
272243
273−/* Map DMA region */
274−static int __vfio_dma_map(struct vfio_iommu *iommu, dma_addr_t iova,
275− unsigned long vaddr, long npage, int prot)
244+/*
245+ * Attempt to pin pages. We really don't want to track all the pfns and
246+ * the iommu can only map chunks of consecutive pfns anyway, so get the
247+ * first page and all consecutive pages with the same locking.
248+ */
249+static long vfio_pin_pages(unsigned long vaddr, long npage,
250+ int prot, unsigned long *pfn_base)
276251 {
277− dma_addr_t start = iova;
278− long i, locked = 0;
279− int ret;
252+ unsigned long limit = rlimit(RLIMIT_MEMLOCK) >> PAGE_SHIFT;
253+ bool lock_cap = capable(CAP_IPC_LOCK);
254+ long ret, i;
280255
281− /* Verify that pages are not already mapped */
282− for (i = 0; i < npage; i++, iova += PAGE_SIZE)
283− if (iommu_iova_to_phys(iommu->domain, iova))
284− return -EBUSY;
256+ if (!current->mm)
257+ return -ENODEV;
285258
286− iova = start;
259+ ret = vaddr_get_pfn(vaddr, prot, pfn_base);
260+ if (ret)
261+ return ret;
287262
288− if (iommu->cache)
289− prot |= IOMMU_CACHE;
263+ if (is_invalid_reserved_pfn(*pfn_base))
264+ return 1;
290265
291− /*
292− * XXX We break mappings into pages and use get_user_pages_fast to
293− * pin the pages in memory. It's been suggested that mlock might
294− * provide a more efficient mechanism, but nothing prevents the
295− * user from munlocking the pages, which could then allow the user
296− * access to random host memory. We also have no guarantee from the
297− * IOMMU API that the iommu driver can unmap sub-pages of previous
298− * mappings. This means we might lose an entire range if a single
299− * page within it is unmapped. Single page mappings are inefficient,
300− * but provide the most flexibility for now.
301− */
302− for (i = 0; i < npage; i++, iova += PAGE_SIZE, vaddr += PAGE_SIZE) {
266+ if (!lock_cap && current->mm->locked_vm + 1 > limit) {
267+ put_pfn(*pfn_base, prot);
268+ pr_warn("%s: RLIMIT_MEMLOCK (%ld) exceeded\n", __func__,
269+ limit << PAGE_SHIFT);
270+ return -ENOMEM;
271+ }
272+
273+ /* Lock all the consecutive pages from pfn_base */
274+ for (i = 1, vaddr += PAGE_SIZE; i < npage; i++, vaddr += PAGE_SIZE) {
303275 unsigned long pfn = 0;
304276
305277 ret = vaddr_get_pfn(vaddr, prot, &pfn);
306− if (ret) {
307− __vfio_dma_do_unmap(iommu, start, i, prot);
308− return ret;
278+ if (ret)
279+ break;
280+
281+ if (pfn != *pfn_base + i || is_invalid_reserved_pfn(pfn)) {
282+ put_pfn(pfn, prot);
283+ break;
309284 }
310285
286+ if (!lock_cap && current->mm->locked_vm + i + 1 > limit) {
287+ put_pfn(pfn, prot);
288+ pr_warn("%s: RLIMIT_MEMLOCK (%ld) exceeded\n",
289+ __func__, limit << PAGE_SHIFT);
290+ break;
291+ }
292+ }
293+
294+ vfio_lock_acct(i);
295+
296+ return i;
297+}
298+
299+static long vfio_unpin_pages(unsigned long pfn, long npage,
300+ int prot, bool do_accounting)
301+{
302+ unsigned long unlocked = 0;
303+ long i;
304+
305+ for (i = 0; i < npage; i++)
306+ unlocked += put_pfn(pfn++, prot);
307+
308+ if (do_accounting)
309+ vfio_lock_acct(-unlocked);
310+
311+ return unlocked;
312+}
313+
314+static int vfio_unmap_unpin(struct vfio_iommu *iommu, struct vfio_dma *dma,
315+ dma_addr_t iova, size_t *size)
316+{
317+ dma_addr_t start = iova, end = iova + *size;
318+ long unlocked = 0;
319+
320+ while (iova < end) {
321+ size_t unmapped;
322+ phys_addr_t phys;
323+
311324 /*
312− * Only add actual locked pages to accounting
313− * XXX We're effectively marking a page locked for every
314− * IOVA page even though it's possible the user could be
315− * backing multiple IOVAs with the same vaddr. This over-
316− * penalizes the user process, but we currently have no
317− * easy way to do this properly.
325+ * We use the IOMMU to track the physical address. This
326+ * saves us from having a lot more entries in our mapping
327+ * tree. The downside is that we don't track the size
328+ * used to do the mapping. We request unmap of a single
329+ * page, but expect IOMMUs that support large pages to
330+ * unmap a larger chunk.
318331 */
319− if (!is_invalid_reserved_pfn(pfn))
320− locked++;
321−
322− ret = iommu_map(iommu->domain, iova,
323− (phys_addr_t)pfn << PAGE_SHIFT,
324− PAGE_SIZE, prot);
325− if (ret) {
326− /* Back out mappings on error */
327− put_pfn(pfn, prot);
328− __vfio_dma_do_unmap(iommu, start, i, prot);
329− return ret;
332+ phys = iommu_iova_to_phys(iommu->domain, iova);
333+ if (WARN_ON(!phys)) {
334+ iova += PAGE_SIZE;
335+ continue;
330336 }
337+
338+ unmapped = iommu_unmap(iommu->domain, iova, PAGE_SIZE);
339+ if (!unmapped)
340+ break;
341+
342+ unlocked += vfio_unpin_pages(phys >> PAGE_SHIFT,
343+ unmapped >> PAGE_SHIFT,
344+ dma->prot, false);
345+ iova += unmapped;
331346 }
332− vfio_lock_acct(locked);
347+
348+ vfio_lock_acct(-unlocked);
349+
350+ *size = iova - start;
351+
333352 return 0;
334353 }
335354
336355 static int vfio_remove_dma_overlap(struct vfio_iommu *iommu, dma_addr_t start,
337− size_t size, struct vfio_dma *dma)
356+ size_t *size, struct vfio_dma *dma)
338357 {
358+ size_t offset, overlap, tmp;
339359 struct vfio_dma *split;
340− long npage_lo, npage_hi;
360+ int ret;
361+
362+ /*
363+ * Existing dma region is completely covered, unmap all. This is
364+ * the likely case since userspace tends to map and unmap buffers
365+ * in one shot rather than multiple mappings within a buffer.
366+ */
367+ if (likely(start <= dma->iova &&
368+ start + *size >= dma->iova + dma->size)) {
369+ *size = dma->size;
370+ ret = vfio_unmap_unpin(iommu, dma, dma->iova, size);
371+ if (ret)
372+ return ret;
373+
374+ /*
375+ * Did we remove more than we have? Should never happen
376+ * since a vfio_dma is contiguous in iova and vaddr.
377+ */
378+ WARN_ON(*size != dma->size);
341379
342− /* Existing dma region is completely covered, unmap all */
343− if (start <= dma->iova &&
344− start + size >= dma->iova + NPAGE_TO_SIZE(dma->npage)) {
345− vfio_dma_unmap(iommu, dma->iova, dma->npage, dma->prot);
346380 vfio_remove_dma(iommu, dma);
347381 kfree(dma);
348382 return 0;
@@ -350,47 +384,79 @@ static int vfio_remove_dma_overlap(struct vfio_iommu *iommu, dma_addr_t start, · 1 unmodified line
350384
351385 /* Overlap low address of existing range */
352386 if (start <= dma->iova) {
353− size_t overlap;
387+ overlap = start + *size - dma->iova;
388+ ret = vfio_unmap_unpin(iommu, dma, dma->iova, &overlap);
389+ if (ret)
390+ return ret;
354391
355− overlap = start + size - dma->iova;
356− npage_lo = overlap >> PAGE_SHIFT;
392+ vfio_remove_dma(iommu, dma);
357393
358− vfio_dma_unmap(iommu, dma->iova, npage_lo, dma->prot);
359− dma->iova += overlap;
360− dma->vaddr += overlap;
361− dma->npage -= npage_lo;
394+ /*
395+ * Check, we may have removed to whole vfio_dma. If not
396+ * fixup and re-insert.
397+ */
398+ if (overlap < dma->size) {
399+ dma->iova += overlap;
400+ dma->vaddr += overlap;
401+ dma->size -= overlap;
402+ vfio_insert_dma(iommu, dma);
403+ }
404+ *size = overlap;
362405 return 0;
363406 }
364407
365408 /* Overlap high address of existing range */
366− if (start + size >= dma->iova + NPAGE_TO_SIZE(dma->npage)) {
367− size_t overlap;
409+ if (start + *size >= dma->iova + dma->size) {
410+ offset = start - dma->iova;
411+ overlap = dma->size - offset;
368412
369− overlap = dma->iova + NPAGE_TO_SIZE(dma->npage) - start;
370− npage_hi = overlap >> PAGE_SHIFT;
413+ ret = vfio_unmap_unpin(iommu, dma, start, &overlap);
414+ if (ret)
415+ return ret;
416+
417+ /*
418+ * We may have unmapped the entire vfio_dma if the user is
419+ * trying to unmap a sub-region of what was originally
420+ * mapped. If anything left, we can resize in place since
421+ * iova is unchanged.
422+ */
423+ if (overlap < dma->size)
424+ dma->size -= overlap;
425+ else
426+ vfio_remove_dma(iommu, dma);
371427
372− vfio_dma_unmap(iommu, start, npage_hi, dma->prot);
373− dma->npage -= npage_hi;
428+ *size = overlap;
374429 return 0;
375430 }
376431
377432 /* Split existing */
378− npage_lo = (start - dma->iova) >> PAGE_SHIFT;
379− npage_hi = dma->npage - (size >> PAGE_SHIFT) - npage_lo;
433+ offset = start - dma->iova;
380434
381− split = kzalloc(sizeof *split, GFP_KERNEL);
382− if (!split)
383− return -ENOMEM;
435+ ret = vfio_unmap_unpin(iommu, dma, start, size);
436+ if (ret)
437+ return ret;
384438
385− vfio_dma_unmap(iommu, start, size >> PAGE_SHIFT, dma->prot);
439+ WARN_ON(!*size);
440+ tmp = dma->size;
386441
387− dma->npage = npage_lo;
442+ /*
443+ * Resize the lower vfio_dma in place, insert new for remaining
444+ * upper segment.
445+ */
446+ dma->size = offset;
447+
448+ if (offset + *size < tmp) {
449+ split = kzalloc(sizeof(*split), GFP_KERNEL);
450+ if (!split)
451+ return -ENOMEM;
452+
453+ split->size = tmp - offset - *size;
454+ split->iova = dma->iova + offset + *size;
455+ split->vaddr = dma->vaddr + offset + *size;
456+ split->prot = dma->prot;
457+ vfio_insert_dma(iommu, split);
458+ }
388459
389− split->npage = npage_hi;
390− split->iova = start + size;
391− split->vaddr = dma->vaddr + NPAGE_TO_SIZE(npage_lo) + size;
392− split->prot = dma->prot;
393− vfio_insert_dma(iommu, split);
394460 return 0;
395461 }
396462
@@ -399,6 +465,7 @@ static int vfio_dma_do_unmap(struct vfio_iommu *iommu, · 2 unmodified lines
399465 {
400466 uint64_t mask;
401467 struct vfio_dma *dma;
468+ size_t unmapped = 0, size;
402469 int ret = 0;
403470
404471 mask = ((uint64_t)1 << __ffs(iommu->domain->ops->pgsize_bitmap)) - 1;
@@ -408,30 +475,66 @@ static int vfio_dma_do_unmap(struct vfio_iommu *iommu, · 3 unmodified lines
408475 if (unmap->size & mask)
409476 return -EINVAL;
410477
411− /* XXX We still break these down into PAGE_SIZE */
412478 WARN_ON(mask & PAGE_MASK);
413479
414480 mutex_lock(&iommu->lock);
415481
416− while (!ret && (dma = vfio_find_dma(iommu,
417− unmap->iova, unmap->size)))
418− ret = vfio_remove_dma_overlap(iommu, unmap->iova,
419− unmap->size, dma);
482+ while ((dma = vfio_find_dma(iommu, unmap->iova, unmap->size))) {
483+ size = unmap->size;
484+ ret = vfio_remove_dma_overlap(iommu, unmap->iova, &size, dma);
485+ if (ret)
486+ break;
487+ unmapped += size;
488+ }
420489
421490 mutex_unlock(&iommu->lock);
491+
492+ /*
493+ * We may unmap more than requested, update the unmap struct so
494+ * userspace can know.
495+ */
496+ unmap->size = unmapped;
497+
498+ return ret;
499+}
500+
501+/*
502+ * Turns out AMD IOMMU has a page table bug where it won't map large pages
503+ * to a region that previously mapped smaller pages. This should be fixed
504+ * soon, so this is just a temporary workaround to break mappings down into
505+ * PAGE_SIZE. Better to map smaller pages than nothing.
506+ */
507+static int map_try_harder(struct vfio_iommu *iommu, dma_addr_t iova,
508+ unsigned long pfn, long npage, int prot)
509+{
510+ long i;
511+ int ret;
512+
513+ for (i = 0; i < npage; i++, pfn++, iova += PAGE_SIZE) {
514+ ret = iommu_map(iommu->domain, iova,
515+ (phys_addr_t)pfn << PAGE_SHIFT,
516+ PAGE_SIZE, prot);
517+ if (ret)
518+ break;
519+ }
520+
521+ for (; i < npage && i > 0; i--, iova -= PAGE_SIZE)
522+ iommu_unmap(iommu->domain, iova, PAGE_SIZE);
523+
422524 return ret;
423525 }
424526
425527 static int vfio_dma_do_map(struct vfio_iommu *iommu,
426528 struct vfio_iommu_type1_dma_map *map)
427529 {
428− struct vfio_dma *dma;
429− dma_addr_t iova = map->iova;
430− unsigned long locked, lock_limit, vaddr = map->vaddr;
530+ dma_addr_t end, iova;
531+ unsigned long vaddr = map->vaddr;
431532 size_t size = map->size;
533+ long npage;
432534 int ret = 0, prot = 0;
433535 uint64_t mask;
434− long npage;
536+
537+ end = map->iova + map->size;
435538
436539 mask = ((uint64_t)1 << __ffs(iommu->domain->ops->pgsize_bitmap)) - 1;
437540
@@ -444,92 +547,138 @@ static int vfio_dma_do_map(struct vfio_iommu *iommu, · 6 unmodified lines
444547 if (!prot)
445548 return -EINVAL; /* No READ/WRITE? */
446549
550+ if (iommu->cache)
551+ prot |= IOMMU_CACHE;
552+
447553 if (vaddr & mask)
448554 return -EINVAL;
449− if (iova & mask)
555+ if (map->iova & mask)
450556 return -EINVAL;
451− if (size & mask)
557+ if (!map->size || map->size & mask)
452558 return -EINVAL;
453559
454− /* XXX We still break these down into PAGE_SIZE */
455560 WARN_ON(mask & PAGE_MASK);
456561
457562 /* Don't allow IOVA wrap */
458− if (iova + size && iova + size < iova)
563+ if (end && end < map->iova)
459564 return -EINVAL;
460565
461566 /* Don't allow virtual address wrap */
462− if (vaddr + size && vaddr + size < vaddr)
463− return -EINVAL;
464−
465− npage = size >> PAGE_SHIFT;
466− if (!npage)
567+ if (vaddr + map->size && vaddr + map->size < vaddr)
467568 return -EINVAL;
468569
469− dma = kzalloc(sizeof *dma, GFP_KERNEL);
470− if (!dma)
471− return -ENOMEM;
472−
473570 mutex_lock(&iommu->lock);
474571
475− if (vfio_find_dma(iommu, iova, size)) {
476− ret = -EBUSY;
477− goto out_lock;
572+ if (vfio_find_dma(iommu, map->iova, map->size)) {
573+ mutex_unlock(&iommu->lock);
574+ return -EEXIST;
478575 }
479576
480− /* account for locked pages */
481− locked = current->mm->locked_vm + npage;
482− lock_limit = rlimit(RLIMIT_MEMLOCK) >> PAGE_SHIFT;
483− if (locked > lock_limit && !capable(CAP_IPC_LOCK)) {
484− pr_warn("%s: RLIMIT_MEMLOCK (%ld) exceeded\n",
485− __func__, rlimit(RLIMIT_MEMLOCK));
486− ret = -ENOMEM;
487− goto out_lock;
488− }
577+ for (iova = map->iova; iova < end; iova += size, vaddr += size) {
578+ struct vfio_dma *dma = NULL;
579+ unsigned long pfn;
580+ long i;
581+
582+ /* Pin a contiguous chunk of memory */
583+ npage = vfio_pin_pages(vaddr, (end - iova) >> PAGE_SHIFT,
584+ prot, &pfn);
585+ if (npage <= 0) {
586+ WARN_ON(!npage);
587+ ret = (int)npage;
588+ break;
589+ }
489590
490− ret = __vfio_dma_map(iommu, iova, vaddr, npage, prot);
491− if (ret)
492− goto out_lock;
493−
494− dma->npage = npage;
495− dma->iova = iova;
496− dma->vaddr = vaddr;
497− dma->prot = prot;
498−
499− /* Check if we abut a region below - nothing below 0 */
500− if (iova) {
501− struct vfio_dma *tmp = vfio_find_dma(iommu, iova - 1, 1);
502− if (tmp && tmp->prot == prot &&
503− tmp->vaddr + NPAGE_TO_SIZE(tmp->npage) == vaddr) {
504− vfio_remove_dma(iommu, tmp);
505− dma->npage += tmp->npage;
506− dma->iova = iova = tmp->iova;
507− dma->vaddr = vaddr = tmp->vaddr;
508− kfree(tmp);
509− npage = dma->npage;
510− size = NPAGE_TO_SIZE(npage);
591+ /* Verify pages are not already mapped */
592+ for (i = 0; i < npage; i++) {
593+ if (iommu_iova_to_phys(iommu->domain,
594+ iova + (i << PAGE_SHIFT))) {
595+ vfio_unpin_pages(pfn, npage, prot, true);
596+ ret = -EBUSY;
597+ break;
598+ }
599+ }
600+
601+ ret = iommu_map(iommu->domain, iova,
602+ (phys_addr_t)pfn << PAGE_SHIFT,
603+ npage << PAGE_SHIFT, prot);
604+ if (ret) {
605+ if (ret != -EBUSY ||
606+ map_try_harder(iommu, iova, pfn, npage, prot)) {
607+ vfio_unpin_pages(pfn, npage, prot, true);
608+ break;
609+ }
610+ }
611+
612+ size = npage << PAGE_SHIFT;
613+
614+ /*
615+ * Check if we abut a region below - nothing below 0.
616+ * This is the most likely case when mapping chunks of
617+ * physically contiguous regions within a virtual address
618+ * range. Update the abutting entry in place since iova
619+ * doesn't change.
620+ */
621+ if (likely(iova)) {
622+ struct vfio_dma *tmp;
623+ tmp = vfio_find_dma(iommu, iova - 1, 1);
624+ if (tmp && tmp->prot == prot &&
625+ tmp->vaddr + tmp->size == vaddr) {
626+ tmp->size += size;
627+
628+ iova = tmp->iova;
629+ size = tmp->size;
630+ vaddr = tmp->vaddr;
631+ dma = tmp;
632+ }
633+ }
634+
635+ /* Check if we abut a region above - nothing above ~0 + 1 */
636+ if (likely(iova + size)) {
637+ struct vfio_dma *tmp;
638+
639+ tmp = vfio_find_dma(iommu, iova + size, 1);
640+ if (tmp && tmp->prot == prot &&
641+ tmp->vaddr == vaddr + size) {
642+ vfio_remove_dma(iommu, tmp);
643+ if (dma)
644+ dma->size += tmp->size;
645+ else
646+ size += tmp->size;
647+ kfree(tmp);
648+ }
511649 }
512− }
513650
514− /* Check if we abut a region above - nothing above ~0 + 1 */
515− if (iova + size) {
516− struct vfio_dma *tmp = vfio_find_dma(iommu, iova + size, 1);
517− if (tmp && tmp->prot == prot &&
518− tmp->vaddr == vaddr + size) {
519− vfio_remove_dma(iommu, tmp);
520− dma->npage += tmp->npage;
521− kfree(tmp);
522− npage = dma->npage;
523− size = NPAGE_TO_SIZE(npage);
651+ if (!dma) {
652+ dma = kzalloc(sizeof(*dma), GFP_KERNEL);
653+ if (!dma) {
654+ iommu_unmap(iommu->domain, iova, size);
655+ vfio_unpin_pages(pfn, npage, prot, true);
656+ ret = -ENOMEM;
657+ break;
658+ }
659+
660+ dma->size = size;
661+ dma->iova = iova;
662+ dma->vaddr = vaddr;
663+ dma->prot = prot;
664+ vfio_insert_dma(iommu, dma);
524665 }
525666 }
526667
527− vfio_insert_dma(iommu, dma);
668+ if (ret) {
669+ struct vfio_dma *tmp;
670+ iova = map->iova;
671+ size = map->size;
672+ while ((tmp = vfio_find_dma(iommu, iova, size))) {
673+ if (vfio_remove_dma_overlap(iommu, iova, &size, tmp)) {
674+ pr_warn("%s: Error rolling back failed map\n",
675+ __func__);
676+ break;
677+ }
678+ }
679+ }
528680
529−out_lock:
530681 mutex_unlock(&iommu->lock);
531− if (ret)
532− kfree(dma);
533682 return ret;
534683 }
535684
@@ -651,9 +800,8 @@ static void vfio_iommu_type1_release(void *iommu_data) · 115 unmodified lines
651800
652801 while ((node = rb_first(&iommu->dma_list))) {
653802 struct vfio_dma *dma = rb_entry(node, struct vfio_dma, node);
654− vfio_dma_unmap(iommu, dma->iova, dma->npage, dma->prot);
655− vfio_remove_dma(iommu, dma);
656− kfree(dma);
803+ size_t size = dma->size;
804+ vfio_remove_dma_overlap(iommu, dma->iova, &size, dma);
657805 }
658806
659807 iommu_domain_free(iommu->domain);
@@ -708,6 +856,7 @@ static long vfio_iommu_type1_ioctl(void *iommu_data, · 48 unmodified lines
708856
709857 } else if (cmd == VFIO_IOMMU_UNMAP_DMA) {
710858 struct vfio_iommu_type1_dma_unmap unmap;
859+ long ret;
711860
712861 minsz = offsetofend(struct vfio_iommu_type1_dma_unmap, size);
713862
@@ -717,7 +866,11 @@ static long vfio_iommu_type1_ioctl(void *iommu_data, · 3 unmodified lines
717866 if (unmap.argsz < minsz || unmap.flags)
718867 return -EINVAL;
719868
720− return vfio_dma_do_unmap(iommu, &unmap);
869+ ret = vfio_dma_do_unmap(iommu, &unmap);
870+ if (ret)
871+ return ret;
872+
873+ return copy_to_user((void __user *)arg, &unmap, minsz);
721874 }
722875
723876 return -ENOTTY;
include/uapi/linux/vfio.h+6−2
@@ -361,10 +361,14 @@ struct vfio_iommu_type1_dma_map { · 360 unmodified lines
361361 #define VFIO_IOMMU_MAP_DMA _IO(VFIO_TYPE, VFIO_BASE + 13)
362362
363363 /**
364− * VFIO_IOMMU_UNMAP_DMA - _IOW(VFIO_TYPE, VFIO_BASE + 14, struct vfio_dma_unmap)
364+ * VFIO_IOMMU_UNMAP_DMA - _IOWR(VFIO_TYPE, VFIO_BASE + 14,
365+ * struct vfio_dma_unmap)
365366 *
366367 * Unmap IO virtual addresses using the provided struct vfio_dma_unmap.
367− * Caller sets argsz.
368+ * Caller sets argsz. The actual unmapped size is returned in the size
369+ * field. No guarantee is made to the user that arbitrary unmaps of iova
370+ * or size different from those used in the original mapping call will
371+ * succeed.
368372 */
369373 struct vfio_iommu_type1_dma_unmap {
370374 __u32 argsz;

Keyboard shortcuts

General

Show keyboard shortcuts
?
Focus the current search
/

Issues and pull requests

Open filters
f

Issues, pull requests and Inbox

Select next item
j↓
Select previous item
k↑
Open selected item
Enter
Dismiss filters or clear selection
Esc

Repository code

Go to file
t