Merge tag 'vfio-v5.12-rc1' of git://github.com/awilliam/linux-vfio

Pull VFIO updatesfrom Alex Williamson: - Virtual address update handling (Steve Sistare) - s390/zpci fixes and cleanups (Max Gurtovoy) - Fixes for dirty bitmap handling, non-mdev page pinning, and improved pinned dirty scope tracking (Keqian Zhu) - Batched page pinning enhancement (Daniel Jordan) - Page access permission fix (Alex Williamson) * tag 'vfio-v5.12-rc1' of git://github.com/awilliam/linux-vfio: (21 commits) vfio/type1: Batch page pinning vfio/type1: Prepare for batched pinning with struct vfio_batch vfio/type1: Change success value of vaddr_get_pfn() vfio/type1: Use follow_pte() vfio/pci: remove CONFIG_VFIO_PCI_ZDEV from Kconfig vfio/iommu_type1: Fix duplicate included kthread.h vfio-pci/zdev: fix possible segmentation fault issue vfio-pci/zdev: remove unused vdev argument vfio/pci: Fix handling of pci use accessor return codes vfio/iommu_type1: Mantain a counter for non_pinned_groups vfio/iommu_type1: Fix some sanity checks in detach group vfio/iommu_type1: Populate full dirty when detach non-pinned group vfio/type1: block on invalid vaddr vfio/type1: implement notify callback vfio: iommu driver notify callback vfio/type1: implement interfaces to update vaddr vfio/type1: massage unmap iteration vfio: interfaces to update vaddr vfio/type1: implement unmap all vfio/type1: unmap cleanup ...

Linus Torvalds committed 5 years ago

Commit 719bbd42 parents c4fbde8 4d83de610 files · +475 −190Browse files

Unified · Split

10 files

drivers/vfio/pci/Kconfig +0 −12
drivers/vfio/pci/Makefile +1 −1
drivers/vfio/pci/vfio_pci.c +5 −7
drivers/vfio/pci/vfio_pci_igd.c +5 −5
drivers/vfio/pci/vfio_pci_private.h +1 −1
drivers/vfio/pci/vfio_pci_zdev.c +12 −12
drivers/vfio/vfio.c +5 −0
drivers/vfio/vfio_iommu_type1.c +412 −152
include/linux/vfio.h +7 −0
include/uapi/linux/vfio.h +27 −0

drivers/vfio/pci/Kconfig+0−12

@@ -45,15 +45,3 @@ config VFIO_PCI_NVLINK2 · 44 unmodified lines
45	45	depends on VFIO_PCI && PPC_POWERNV
46	46	help
47	47	VFIO PCI support for P9 Witherspoon machine with NVIDIA V100 GPUs
48		−
49		−config VFIO_PCI_ZDEV
50		− bool "VFIO PCI ZPCI device CLP support"
51		− depends on VFIO_PCI && S390
52		− default y
53		− help
54		− Enabling this option exposes VFIO capabilities containing hardware
55		− configuration for zPCI devices. This enables userspace (e.g. QEMU)
56		− to supply proper configuration values instead of hard-coded defaults
57		− for zPCI devices passed through via VFIO on s390.
58		−
59		− Say Y here.

drivers/vfio/pci/Makefile+1−1

@@ -3,6 +3,6 @@ · 2 unmodified lines
3	3	vfio-pci-y := vfio_pci.o vfio_pci_intrs.o vfio_pci_rdwr.o vfio_pci_config.o
4	4	vfio-pci-$(CONFIG_VFIO_PCI_IGD) += vfio_pci_igd.o
5	5	vfio-pci-$(CONFIG_VFIO_PCI_NVLINK2) += vfio_pci_nvlink2.o
6		−vfio-pci-$(CONFIG_VFIO_PCI_ZDEV) += vfio_pci_zdev.o
	6	+vfio-pci-$(CONFIG_S390) += vfio_pci_zdev.o
7	7
8	8	obj-$(CONFIG_VFIO_PCI) += vfio-pci.o

drivers/vfio/pci/vfio_pci.c+5−7

@@ -807,6 +807,7 @@ static long vfio_pci_ioctl(void *device_data, · 806 unmodified lines
807	807	struct vfio_device_info info;
808	808	struct vfio_info_cap caps = { .buf = NULL, .size = 0 };
809	809	unsigned long capsz;
	810	+ int ret;
810	811
811	812	minsz = offsetofend(struct vfio_device_info, num_irqs);
812	813
@@ -832,13 +833,10 @@ static long vfio_pci_ioctl(void *device_data, · 19 unmodified lines
832	833	info.num_regions = VFIO_PCI_NUM_REGIONS + vdev->num_regions;
833	834	info.num_irqs = VFIO_PCI_NUM_IRQS;
834	835
835		− if (IS_ENABLED(CONFIG_VFIO_PCI_ZDEV)) {
836		− int ret = vfio_pci_info_zdev_add_caps(vdev, &caps);
837		−
838		− if (ret && ret != -ENODEV) {
839		− pci_warn(vdev->pdev, "Failed to setup zPCI info capabilities\n");
840		− return ret;
841		− }
	836	+ ret = vfio_pci_info_zdev_add_caps(vdev, &caps);
	837	+ if (ret && ret != -ENODEV) {
	838	+ pci_warn(vdev->pdev, "Failed to setup zPCI info capabilities\n");
	839	+ return ret;
842	840	}
843	841
844	842	if (caps.size) {

drivers/vfio/pci/vfio_pci_igd.c+5−5

@@ -127,7 +127,7 @@ static size_t vfio_pci_igd_cfg_rw(struct vfio_pci_device *vdev, · 126 unmodified lines
127	127
128	128	ret = pci_user_read_config_byte(pdev, pos, &val);
129	129	if (ret)
130		− return pcibios_err_to_errno(ret);
	130	+ return ret;
131	131
132	132	if (copy_to_user(buf + count - size, &val, 1))
133	133	return -EFAULT;
@@ -141,7 +141,7 @@ static size_t vfio_pci_igd_cfg_rw(struct vfio_pci_device *vdev, · 7 unmodified lines
141	141
142	142	ret = pci_user_read_config_word(pdev, pos, &val);
143	143	if (ret)
144		− return pcibios_err_to_errno(ret);
	144	+ return ret;
145	145
146	146	val = cpu_to_le16(val);
147	147	if (copy_to_user(buf + count - size, &val, 2))
@@ -156,7 +156,7 @@ static size_t vfio_pci_igd_cfg_rw(struct vfio_pci_device *vdev, · 8 unmodified lines
156	156
157	157	ret = pci_user_read_config_dword(pdev, pos, &val);
158	158	if (ret)
159		− return pcibios_err_to_errno(ret);
	159	+ return ret;
160	160
161	161	val = cpu_to_le32(val);
162	162	if (copy_to_user(buf + count - size, &val, 4))
@@ -171,7 +171,7 @@ static size_t vfio_pci_igd_cfg_rw(struct vfio_pci_device *vdev, · 8 unmodified lines
171	171
172	172	ret = pci_user_read_config_word(pdev, pos, &val);
173	173	if (ret)
174		− return pcibios_err_to_errno(ret);
	174	+ return ret;
175	175
176	176	val = cpu_to_le16(val);
177	177	if (copy_to_user(buf + count - size, &val, 2))
@@ -186,7 +186,7 @@ static size_t vfio_pci_igd_cfg_rw(struct vfio_pci_device *vdev, · 8 unmodified lines
186	186
187	187	ret = pci_user_read_config_byte(pdev, pos, &val);
188	188	if (ret)
189		− return pcibios_err_to_errno(ret);
	189	+ return ret;
190	190
191	191	if (copy_to_user(buf + count - size, &val, 1))
192	192	return -EFAULT;

drivers/vfio/pci/vfio_pci_private.h+1−1

@@ -214,7 +214,7 @@ static inline int vfio_pci_ibm_npu2_init(struct vfio_pci_device *vdev) · 213 unmodified lines
214	214	}
215	215	#endif
216	216
217		−#ifdef CONFIG_VFIO_PCI_ZDEV
	217	+#ifdef CONFIG_S390
218	218	extern int vfio_pci_info_zdev_add_caps(struct vfio_pci_device *vdev,
219	219	struct vfio_info_cap *caps);
220	220	#else

drivers/vfio/pci/vfio_pci_zdev.c+12−12

@@ -24,8 +24,7 @@ · 23 unmodified lines
24	24	/*
25	25	* Add the Base PCI Function information to the device info region.
26	26	*/
27		−static int zpci_base_cap(struct zpci_dev zdev, struct vfio_pci_device vdev,
28		− struct vfio_info_cap *caps)
	27	+static int zpci_base_cap(struct zpci_dev zdev, struct vfio_info_cap caps)
29	28	{
30	29	struct vfio_device_info_cap_zpci_base cap = {
31	30	.header.id = VFIO_DEVICE_INFO_CAP_ZPCI_BASE,
@@ -45,8 +44,7 @@ static int zpci_base_cap(struct zpci_dev zdev, struct vfio_pci_device vdev, · 13 unmodified lines
45	44	/*
46	45	* Add the Base PCI Function Group information to the device info region.
47	46	*/
48		−static int zpci_group_cap(struct zpci_dev zdev, struct vfio_pci_device vdev,
49		− struct vfio_info_cap *caps)
	47	+static int zpci_group_cap(struct zpci_dev zdev, struct vfio_info_cap caps)
50	48	{
51	49	struct vfio_device_info_cap_zpci_group cap = {
52	50	.header.id = VFIO_DEVICE_INFO_CAP_ZPCI_GROUP,
@@ -66,14 +64,15 @@ static int zpci_group_cap(struct zpci_dev zdev, struct vfio_pci_device vdev, · 13 unmodified lines
66	64	/*
67	65	* Add the device utility string to the device info region.
68	66	*/
69		−static int zpci_util_cap(struct zpci_dev zdev, struct vfio_pci_device vdev,
70		− struct vfio_info_cap *caps)
	67	+static int zpci_util_cap(struct zpci_dev zdev, struct vfio_info_cap caps)
71	68	{
72	69	struct vfio_device_info_cap_zpci_util *cap;
73	70	int cap_size = sizeof(*cap) + CLP_UTIL_STR_LEN;
74	71	int ret;
75	72
76	73	cap = kmalloc(cap_size, GFP_KERNEL);
	74	+ if (!cap)
	75	+ return -ENOMEM;
77	76
78	77	cap->header.id = VFIO_DEVICE_INFO_CAP_ZPCI_UTIL;
79	78	cap->header.version = 1;
@@ -90,14 +89,15 @@ static int zpci_util_cap(struct zpci_dev zdev, struct vfio_pci_device vdev, · 10 unmodified lines
90	89	/*
91	90	* Add the function path string to the device info region.
92	91	*/
93		−static int zpci_pfip_cap(struct zpci_dev zdev, struct vfio_pci_device vdev,
94		− struct vfio_info_cap *caps)
	92	+static int zpci_pfip_cap(struct zpci_dev zdev, struct vfio_info_cap caps)
95	93	{
96	94	struct vfio_device_info_cap_zpci_pfip *cap;
97	95	int cap_size = sizeof(*cap) + CLP_PFIP_NR_SEGMENTS;
98	96	int ret;
99	97
100	98	cap = kmalloc(cap_size, GFP_KERNEL);
	99	+ if (!cap)
	100	+ return -ENOMEM;
101	101
102	102	cap->header.id = VFIO_DEVICE_INFO_CAP_ZPCI_PFIP;
103	103	cap->header.version = 1;
@@ -123,21 +123,21 @@ int vfio_pci_info_zdev_add_caps(struct vfio_pci_device *vdev, · 19 unmodified lines
123	123	if (!zdev)
124	124	return -ENODEV;
125	125
126		− ret = zpci_base_cap(zdev, vdev, caps);
	126	+ ret = zpci_base_cap(zdev, caps);
127	127	if (ret)
128	128	return ret;
129	129
130		− ret = zpci_group_cap(zdev, vdev, caps);
	130	+ ret = zpci_group_cap(zdev, caps);
131	131	if (ret)
132	132	return ret;
133	133
134	134	if (zdev->util_str_avail) {
135		− ret = zpci_util_cap(zdev, vdev, caps);
	135	+ ret = zpci_util_cap(zdev, caps);
136	136	if (ret)
137	137	return ret;
138	138	}
139	139
140		− ret = zpci_pfip_cap(zdev, vdev, caps);
	140	+ ret = zpci_pfip_cap(zdev, caps);
141	141
142	142	return ret;
143	143	}

drivers/vfio/vfio.c+5−0

@@ -1220,6 +1220,11 @@ static int vfio_fops_open(struct inode inode, struct file filep) · 1219 unmodified lines
1220	1220	static int vfio_fops_release(struct inode inode, struct file filep)
1221	1221	{
1222	1222	struct vfio_container *container = filep->private_data;
	1223	+ struct vfio_iommu_driver *driver = container->iommu_driver;
	1224	+
	1225	+ if (driver && driver->ops->notify)
	1226	+ driver->ops->notify(container->iommu_data,
	1227	+ VFIO_IOMMU_CONTAINER_CLOSE);
1223	1228
1224	1229	filep->private_data = NULL;
1225	1230

drivers/vfio/vfio_iommu_type1.c+412−152

@@ -24,6 +24,7 @@ · 23 unmodified lines
24	24	#include <linux/compat.h>
25	25	#include <linux/device.h>
26	26	#include <linux/fs.h>
	27	+#include <linux/highmem.h>
27	28	#include <linux/iommu.h>
28	29	#include <linux/module.h>
29	30	#include <linux/mm.h>
@@ -69,11 +70,15 @@ struct vfio_iommu { · 39 unmodified lines
69	70	struct rb_root dma_list;
70	71	struct blocking_notifier_head notifier;
71	72	unsigned int dma_avail;
	73	+ unsigned int vaddr_invalid_count;
72	74	uint64_t pgsize_bitmap;
	75	+ uint64_t num_non_pinned_groups;
	76	+ wait_queue_head_t vaddr_wait;
73	77	bool v2;
74	78	bool nesting;
75	79	bool dirty_page_tracking;
76	80	bool pinned_page_dirty_scope;
	81	+ bool container_open;
77	82	};
78	83
79	84	struct vfio_domain {
@@ -92,11 +97,20 @@ struct vfio_dma { · 12 unmodified lines
92	97	int prot; /* IOMMU_READ/WRITE */
93	98	bool iommu_mapped;
94	99	bool lock_cap; /* capable(CAP_IPC_LOCK) */
	100	+ bool vaddr_invalid;
95	101	struct task_struct *task;
96	102	struct rb_root pfn_list; /* Ex-user pinned pfn list */
97	103	unsigned long *bitmap;
98	104	};
99	105
	106	+struct vfio_batch {
	107	+ struct page *pages; / for pin_user_pages_remote */
	108	+ struct page fallback_page; / if pages alloc fails */
	109	+ int capacity; /* length of pages array */
	110	+ int size; /* of batch currently */
	111	+ int offset; /* of next entry in pages */
	112	+};
	113	+
100	114	struct vfio_group {
101	115	struct iommu_group *iommu_group;
102	116	struct list_head next;
@@ -143,12 +157,13 @@ struct vfio_regions { · 40 unmodified lines
143	157	#define DIRTY_BITMAP_PAGES_MAX ((u64)INT_MAX)
144	158	#define DIRTY_BITMAP_SIZE_MAX DIRTY_BITMAP_BYTES(DIRTY_BITMAP_PAGES_MAX)
145	159
	160	+#define WAITED 1
	161	+
146	162	static int put_pfn(unsigned long pfn, int prot);
147	163
148	164	static struct vfio_group vfio_iommu_find_iommu_group(struct vfio_iommu iommu,
149	165	struct iommu_group *iommu_group);
150	166
151		−static void update_pinned_page_dirty_scope(struct vfio_iommu *iommu);
152	167	/*
153	168	* This code handles mapping and unmapping of user data buffers
154	169	* into DMA'ble space using the IOMMU
@@ -173,6 +188,31 @@ static struct vfio_dma vfio_find_dma(struct vfio_iommu iommu, · 18 unmodified lines
173	188	return NULL;
174	189	}
175	190
	191	+static struct rb_node vfio_find_dma_first_node(struct vfio_iommu iommu,
	192	+ dma_addr_t start, size_t size)
	193	+{
	194	+ struct rb_node *res = NULL;
	195	+ struct rb_node *node = iommu->dma_list.rb_node;
	196	+ struct vfio_dma *dma_res = NULL;
	197	+
	198	+ while (node) {
	199	+ struct vfio_dma *dma = rb_entry(node, struct vfio_dma, node);
	200	+
	201	+ if (start < dma->iova + dma->size) {
	202	+ res = node;
	203	+ dma_res = dma;
	204	+ if (start >= dma->iova)
	205	+ break;
	206	+ node = node->rb_left;
	207	+ } else {
	208	+ node = node->rb_right;
	209	+ }
	210	+ }
	211	+ if (res && size && dma_res->iova >= start + size)
	212	+ res = NULL;
	213	+ return res;
	214	+}
	215	+
176	216	static void vfio_link_dma(struct vfio_iommu iommu, struct vfio_dma new)
177	217	{
178	218	struct rb_node *link = &iommu->dma_list.rb_node, parent = NULL;
@@ -236,6 +276,18 @@ static void vfio_dma_populate_bitmap(struct vfio_dma *dma, size_t pgsize) · 57 unmodified lines
236	276	}
237	277	}
238	278
	279	+static void vfio_iommu_populate_bitmap_full(struct vfio_iommu *iommu)
	280	+{
	281	+ struct rb_node *n;
	282	+ unsigned long pgshift = __ffs(iommu->pgsize_bitmap);
	283	+
	284	+ for (n = rb_first(&iommu->dma_list); n; n = rb_next(n)) {
	285	+ struct vfio_dma *dma = rb_entry(n, struct vfio_dma, node);
	286	+
	287	+ bitmap_set(dma->bitmap, 0, dma->size >> pgshift);
	288	+ }
	289	+}
	290	+
239	291	static int vfio_dma_bitmap_alloc_all(struct vfio_iommu *iommu, size_t pgsize)
240	292	{
241	293	struct rb_node *n;
@@ -415,13 +467,54 @@ static int put_pfn(unsigned long pfn, int prot) · 173 unmodified lines
415	467	return 0;
416	468	}
417	469
	470	+#define VFIO_BATCH_MAX_CAPACITY (PAGE_SIZE / sizeof(struct page *))
	471	+
	472	+static void vfio_batch_init(struct vfio_batch *batch)
	473	+{
	474	+ batch->size = 0;
	475	+ batch->offset = 0;
	476	+
	477	+ if (unlikely(disable_hugepages))
	478	+ goto fallback;
	479	+
	480	+ batch->pages = (struct page **) __get_free_page(GFP_KERNEL);
	481	+ if (!batch->pages)
	482	+ goto fallback;
	483	+
	484	+ batch->capacity = VFIO_BATCH_MAX_CAPACITY;
	485	+ return;
	486	+
	487	+fallback:
	488	+ batch->pages = &batch->fallback_page;
	489	+ batch->capacity = 1;
	490	+}
	491	+
	492	+static void vfio_batch_unpin(struct vfio_batch batch, struct vfio_dma dma)
	493	+{
	494	+ while (batch->size) {
	495	+ unsigned long pfn = page_to_pfn(batch->pages[batch->offset]);
	496	+
	497	+ put_pfn(pfn, dma->prot);
	498	+ batch->offset++;
	499	+ batch->size--;
	500	+ }
	501	+}
	502	+
	503	+static void vfio_batch_fini(struct vfio_batch *batch)
	504	+{
	505	+ if (batch->capacity == VFIO_BATCH_MAX_CAPACITY)
	506	+ free_page((unsigned long)batch->pages);
	507	+}
	508	+
418	509	static int follow_fault_pfn(struct vm_area_struct vma, struct mm_struct mm,
419	510	unsigned long vaddr, unsigned long *pfn,
420	511	bool write_fault)
421	512	{
	513	+ pte_t *ptep;
	514	+ spinlock_t *ptl;
422	515	int ret;
423	516
424		− ret = follow_pfn(vma, vaddr, pfn);
	517	+ ret = follow_pte(vma->vm_mm, vaddr, &ptep, &ptl);
425	518	if (ret) {
426	519	bool unlocked = false;
427	520
@@ -435,16 +528,28 @@ static int follow_fault_pfn(struct vm_area_struct vma, struct mm_struct mm, · 7 unmodified lines
435	528	if (ret)
436	529	return ret;
437	530
438		− ret = follow_pfn(vma, vaddr, pfn);
	531	+ ret = follow_pte(vma->vm_mm, vaddr, &ptep, &ptl);
	532	+ if (ret)
	533	+ return ret;
439	534	}
440	535
	536	+ if (write_fault && !pte_write(*ptep))
	537	+ ret = -EFAULT;
	538	+ else
	539	+ pfn = pte_pfn(ptep);
	540	+
	541	+ pte_unmap_unlock(ptep, ptl);
441	542	return ret;
442	543	}
443	544
444		−static int vaddr_get_pfn(struct mm_struct *mm, unsigned long vaddr,
445		− int prot, unsigned long *pfn)
	545	+/*
	546	+ * Returns the positive number of pfns successfully obtained or a negative
	547	+ * error code.
	548	+ */
	549	+static int vaddr_get_pfns(struct mm_struct *mm, unsigned long vaddr,
	550	+ long npages, int prot, unsigned long *pfn,
	551	+ struct page **pages)
446	552	{
447		− struct page *page[1];
448	553	struct vm_area_struct *vma;
449	554	unsigned int flags = 0;
450	555	int ret;
@@ -453,11 +558,10 @@ static int vaddr_get_pfn(struct mm_struct *mm, unsigned long vaddr, · 2 unmodified lines
453	558	flags \|= FOLL_WRITE;
454	559
455	560	mmap_read_lock(mm);
456		− ret = pin_user_pages_remote(mm, vaddr, 1, flags \| FOLL_LONGTERM,
457		− page, NULL, NULL);
458		− if (ret == 1) {
459		− *pfn = page_to_pfn(page[0]);
460		− ret = 0;
	561	+ ret = pin_user_pages_remote(mm, vaddr, npages, flags \| FOLL_LONGTERM,
	562	+ pages, NULL, NULL);
	563	+ if (ret > 0) {
	564	+ *pfn = page_to_pfn(pages[0]);
461	565	goto done;
462	566	}
463	567
@@ -471,14 +575,73 @@ retry: · 7 unmodified lines
471	575	if (ret == -EAGAIN)
472	576	goto retry;
473	577
474		− if (!ret && !is_invalid_reserved_pfn(*pfn))
475		− ret = -EFAULT;
	578	+ if (!ret) {
	579	+ if (is_invalid_reserved_pfn(*pfn))
	580	+ ret = 1;
	581	+ else
	582	+ ret = -EFAULT;
	583	+ }
476	584	}
477	585	done:
478	586	mmap_read_unlock(mm);
479	587	return ret;
480	588	}
481	589
	590	+static int vfio_wait(struct vfio_iommu *iommu)
	591	+{
	592	+ DEFINE_WAIT(wait);
	593	+
	594	+ prepare_to_wait(&iommu->vaddr_wait, &wait, TASK_KILLABLE);
	595	+ mutex_unlock(&iommu->lock);
	596	+ schedule();
	597	+ mutex_lock(&iommu->lock);
	598	+ finish_wait(&iommu->vaddr_wait, &wait);
	599	+ if (kthread_should_stop() \|\| !iommu->container_open \|\|
	600	+ fatal_signal_pending(current)) {
	601	+ return -EFAULT;
	602	+ }
	603	+ return WAITED;
	604	+}
	605	+
	606	+/*
	607	+ * Find dma struct and wait for its vaddr to be valid. iommu lock is dropped
	608	+ * if the task waits, but is re-locked on return. Return result in *dma_p.
	609	+ * Return 0 on success with no waiting, WAITED on success if waited, and -errno
	610	+ * on error.
	611	+ */
	612	+static int vfio_find_dma_valid(struct vfio_iommu *iommu, dma_addr_t start,
	613	+ size_t size, struct vfio_dma **dma_p)
	614	+{
	615	+ int ret;
	616	+
	617	+ do {
	618	+ *dma_p = vfio_find_dma(iommu, start, size);
	619	+ if (!*dma_p)
	620	+ ret = -EINVAL;
	621	+ else if (!(*dma_p)->vaddr_invalid)
	622	+ ret = 0;
	623	+ else
	624	+ ret = vfio_wait(iommu);
	625	+ } while (ret > 0);
	626	+
	627	+ return ret;
	628	+}
	629	+
	630	+/*
	631	+ * Wait for all vaddr in the dma_list to become valid. iommu lock is dropped
	632	+ * if the task waits, but is re-locked on return. Return 0 on success with no
	633	+ * waiting, WAITED on success if waited, and -errno on error.
	634	+ */
	635	+static int vfio_wait_all_valid(struct vfio_iommu *iommu)
	636	+{
	637	+ int ret = 0;
	638	+
	639	+ while (iommu->vaddr_invalid_count && ret >= 0)
	640	+ ret = vfio_wait(iommu);
	641	+
	642	+ return ret;
	643	+}
	644	+
482	645	/*
483	646	* Attempt to pin pages. We really don't want to track all the pfns and
484	647	* the iommu can only map chunks of consecutive pfns anyway, so get the
@@ -486,76 +649,102 @@ done: · 1 unmodified line
486	649	*/
487	650	static long vfio_pin_pages_remote(struct vfio_dma *dma, unsigned long vaddr,
488	651	long npage, unsigned long *pfn_base,
489		− unsigned long limit)
	652	+ unsigned long limit, struct vfio_batch *batch)
490	653	{
491		− unsigned long pfn = 0;
	654	+ unsigned long pfn;
	655	+ struct mm_struct *mm = current->mm;
492	656	long ret, pinned = 0, lock_acct = 0;
493	657	bool rsvd;
494	658	dma_addr_t iova = vaddr - dma->vaddr + dma->iova;
495	659
496	660	/* This code path is only user initiated */
497		− if (!current->mm)
	661	+ if (!mm)
498	662	return -ENODEV;
499	663
500		− ret = vaddr_get_pfn(current->mm, vaddr, dma->prot, pfn_base);
501		− if (ret)
502		− return ret;
503		−
504		− pinned++;
505		− rsvd = is_invalid_reserved_pfn(*pfn_base);
506		−
507		− /*
508		− * Reserved pages aren't counted against the user, externally pinned
509		− * pages are already counted against the user.
510		− */
511		− if (!rsvd && !vfio_find_vpfn(dma, iova)) {
512		− if (!dma->lock_cap && current->mm->locked_vm + 1 > limit) {
513		− put_pfn(*pfn_base, dma->prot);
514		− pr_warn("%s: RLIMIT_MEMLOCK (%ld) exceeded\n", __func__,
515		− limit << PAGE_SHIFT);
516		− return -ENOMEM;
517		− }
518		− lock_acct++;
	664	+ if (batch->size) {
	665	+ /* Leftover pages in batch from an earlier call. */
	666	+ *pfn_base = page_to_pfn(batch->pages[batch->offset]);
	667	+ pfn = *pfn_base;
	668	+ rsvd = is_invalid_reserved_pfn(*pfn_base);
	669	+ } else {
	670	+ *pfn_base = 0;
519	671	}
520	672
521		− if (unlikely(disable_hugepages))
522		− goto out;
	673	+ while (npage) {
	674	+ if (!batch->size) {
	675	+ /* Empty batch, so refill it. */
	676	+ long req_pages = min_t(long, npage, batch->capacity);
523	677
524		− /* Lock all the consecutive pages from pfn_base */
525		− for (vaddr += PAGE_SIZE, iova += PAGE_SIZE; pinned < npage;
526		− pinned++, vaddr += PAGE_SIZE, iova += PAGE_SIZE) {
527		− ret = vaddr_get_pfn(current->mm, vaddr, dma->prot, &pfn);
528		− if (ret)
529		− break;
	678	+ ret = vaddr_get_pfns(mm, vaddr, req_pages, dma->prot,
	679	+ &pfn, batch->pages);
	680	+ if (ret < 0)
	681	+ goto unpin_out;
530	682
531		− if (pfn != *pfn_base + pinned \|\|
532		− rsvd != is_invalid_reserved_pfn(pfn)) {
533		− put_pfn(pfn, dma->prot);
534		− break;
	683	+ batch->size = ret;
	684	+ batch->offset = 0;
	685	+
	686	+ if (!*pfn_base) {
	687	+ *pfn_base = pfn;
	688	+ rsvd = is_invalid_reserved_pfn(*pfn_base);
	689	+ }
535	690	}
536	691
537		− if (!rsvd && !vfio_find_vpfn(dma, iova)) {
538		− if (!dma->lock_cap &&
539		− current->mm->locked_vm + lock_acct + 1 > limit) {
540		− put_pfn(pfn, dma->prot);
541		− pr_warn("%s: RLIMIT_MEMLOCK (%ld) exceeded\n",
542		− __func__, limit << PAGE_SHIFT);
543		− ret = -ENOMEM;
544		− goto unpin_out;
	692	+ /*
	693	+ * pfn is preset for the first iteration of this inner loop and
	694	+ * updated at the end to handle a VM_PFNMAP pfn. In that case,
	695	+ * batch->pages isn't valid (there's no struct page), so allow
	696	+ * batch->pages to be touched only when there's more than one
	697	+ * pfn to check, which guarantees the pfns are from a
	698	+ * !VM_PFNMAP vma.
	699	+ */
	700	+ while (true) {
	701	+ if (pfn != *pfn_base + pinned \|\|
	702	+ rsvd != is_invalid_reserved_pfn(pfn))
	703	+ goto out;
	704	+
	705	+ /*
	706	+ * Reserved pages aren't counted against the user,
	707	+ * externally pinned pages are already counted against
	708	+ * the user.
	709	+ */
	710	+ if (!rsvd && !vfio_find_vpfn(dma, iova)) {
	711	+ if (!dma->lock_cap &&
	712	+ mm->locked_vm + lock_acct + 1 > limit) {
	713	+ pr_warn("%s: RLIMIT_MEMLOCK (%ld) exceeded\n",
	714	+ __func__, limit << PAGE_SHIFT);
	715	+ ret = -ENOMEM;
	716	+ goto unpin_out;
	717	+ }
	718	+ lock_acct++;
545	719	}
546		− lock_acct++;
	720	+
	721	+ pinned++;
	722	+ npage--;
	723	+ vaddr += PAGE_SIZE;
	724	+ iova += PAGE_SIZE;
	725	+ batch->offset++;
	726	+ batch->size--;
	727	+
	728	+ if (!batch->size)
	729	+ break;
	730	+
	731	+ pfn = page_to_pfn(batch->pages[batch->offset]);
547	732	}
	733	+
	734	+ if (unlikely(disable_hugepages))
	735	+ break;
548	736	}
549	737
550	738	out:
551	739	ret = vfio_lock_acct(dma, lock_acct, false);
552	740
553	741	unpin_out:
554		− if (ret) {
555		− if (!rsvd) {
	742	+ if (ret < 0) {
	743	+ if (pinned && !rsvd) {
556	744	for (pfn = *pfn_base ; pinned ; pfn++, pinned--)
557	745	put_pfn(pfn, dma->prot);
558	746	}
	747	+ vfio_batch_unpin(batch, dma);
559	748
560	749	return ret;
561	750	}
@@ -587,6 +776,7 @@ static long vfio_unpin_pages_remote(struct vfio_dma *dma, dma_addr_t iova, · 25 unmodified lines
587	776	static int vfio_pin_page_external(struct vfio_dma *dma, unsigned long vaddr,
588	777	unsigned long *pfn_base, bool do_accounting)
589	778	{
	779	+ struct page *pages[1];
590	780	struct mm_struct *mm;
591	781	int ret;
592	782
@@ -594,8 +784,8 @@ static int vfio_pin_page_external(struct vfio_dma *dma, unsigned long vaddr, · 1 unmodified line
594	784	if (!mm)
595	785	return -ENODEV;
596	786
597		− ret = vaddr_get_pfn(mm, vaddr, dma->prot, pfn_base);
598		− if (!ret && do_accounting && !is_invalid_reserved_pfn(*pfn_base)) {
	787	+ ret = vaddr_get_pfns(mm, vaddr, 1, dma->prot, pfn_base, pages);
	788	+ if (ret == 1 && do_accounting && !is_invalid_reserved_pfn(*pfn_base)) {
599	789	ret = vfio_lock_acct(dma, 1, true);
600	790	if (ret) {
601	791	put_pfn(*pfn_base, dma->prot);
@@ -640,6 +830,7 @@ static int vfio_iommu_type1_pin_pages(void *iommu_data, · 38 unmodified lines
640	830	unsigned long remote_vaddr;
641	831	struct vfio_dma *dma;
642	832	bool do_accounting;
	833	+ dma_addr_t iova;
643	834
644	835	if (!iommu \|\| !user_pfn \|\| !phys_pfn)
645	836	return -EINVAL;
@@ -650,6 +841,22 @@ static int vfio_iommu_type1_pin_pages(void *iommu_data, · 4 unmodified lines
650	841
651	842	mutex_lock(&iommu->lock);
652	843
	844	+ /*
	845	+ * Wait for all necessary vaddr's to be valid so they can be used in
	846	+ * the main loop without dropping the lock, to avoid racing vs unmap.
	847	+ */
	848	+again:
	849	+ if (iommu->vaddr_invalid_count) {
	850	+ for (i = 0; i < npage; i++) {
	851	+ iova = user_pfn[i] << PAGE_SHIFT;
	852	+ ret = vfio_find_dma_valid(iommu, iova, PAGE_SIZE, &dma);
	853	+ if (ret < 0)
	854	+ goto pin_done;
	855	+ if (ret == WAITED)
	856	+ goto again;
	857	+ }
	858	+ }
	859	+
653	860	/* Fail if notifier list is empty */
654	861	if (!iommu->notifier.head) {
655	862	ret = -EINVAL;
@@ -664,7 +871,6 @@ static int vfio_iommu_type1_pin_pages(void *iommu_data, · 8 unmodified lines
664	871	do_accounting = !IS_IOMMU_CAP_DOMAIN_IN_CONTAINER(iommu);
665	872
666	873	for (i = 0; i < npage; i++) {
667		− dma_addr_t iova;
668	874	struct vfio_pfn *vpfn;
669	875
670	876	iova = user_pfn[i] << PAGE_SHIFT;
@@ -714,7 +920,7 @@ static int vfio_iommu_type1_pin_pages(void *iommu_data, · 43 unmodified lines
714	920	group = vfio_iommu_find_iommu_group(iommu, iommu_group);
715	921	if (!group->pinned_page_dirty_scope) {
716	922	group->pinned_page_dirty_scope = true;
717		− update_pinned_page_dirty_scope(iommu);
	923	+ iommu->num_non_pinned_groups--;
718	924	}
719	925
720	926	goto pin_done;
@@ -945,10 +1151,15 @@ static long vfio_unmap_unpin(struct vfio_iommu iommu, struct vfio_dma dma, · 224 unmodified lines
945	1151
946	1152	static void vfio_remove_dma(struct vfio_iommu iommu, struct vfio_dma dma)
947	1153	{
	1154	+ WARN_ON(!RB_EMPTY_ROOT(&dma->pfn_list));
948	1155	vfio_unmap_unpin(iommu, dma, true);
949	1156	vfio_unlink_dma(iommu, dma);
950	1157	put_task_struct(dma->task);
951	1158	vfio_dma_bitmap_free(dma);
	1159	+ if (dma->vaddr_invalid) {
	1160	+ iommu->vaddr_invalid_count--;
	1161	+ wake_up_all(&iommu->vaddr_wait);
	1162	+ }
952	1163	kfree(dma);
953	1164	iommu->dma_avail++;
954	1165	}
@@ -991,7 +1202,7 @@ static int update_user_bitmap(u64 __user bitmap, struct vfio_iommu iommu, · 36 unmodified lines
991	1202	* mark all pages dirty if any IOMMU capable device is not able
992	1203	* to report dirty pages and all pages are pinned and mapped.
993	1204	*/
994		− if (!iommu->pinned_page_dirty_scope && dma->iommu_mapped)
	1205	+ if (iommu->num_non_pinned_groups && dma->iommu_mapped)
995	1206	bitmap_set(dma->bitmap, 0, nbits);
996	1207
997	1208	if (shift) {
@@ -1074,34 +1285,36 @@ static int vfio_dma_do_unmap(struct vfio_iommu *iommu, · 76 unmodified lines
1074	1285	{
1075	1286	struct vfio_dma dma, dma_last = NULL;
1076	1287	size_t unmapped = 0, pgsize;
1077		− int ret = 0, retries = 0;
	1288	+ int ret = -EINVAL, retries = 0;
1078	1289	unsigned long pgshift;
	1290	+ dma_addr_t iova = unmap->iova;
	1291	+ unsigned long size = unmap->size;
	1292	+ bool unmap_all = unmap->flags & VFIO_DMA_UNMAP_FLAG_ALL;
	1293	+ bool invalidate_vaddr = unmap->flags & VFIO_DMA_UNMAP_FLAG_VADDR;
	1294	+ struct rb_node n, first_n;
1079	1295
1080	1296	mutex_lock(&iommu->lock);
1081	1297
1082	1298	pgshift = __ffs(iommu->pgsize_bitmap);
1083	1299	pgsize = (size_t)1 << pgshift;
1084	1300
1085		− if (unmap->iova & (pgsize - 1)) {
1086		− ret = -EINVAL;
	1301	+ if (iova & (pgsize - 1))
1087	1302	goto unlock;
1088		− }
1089	1303
1090		− if (!unmap->size \|\| unmap->size & (pgsize - 1)) {
1091		− ret = -EINVAL;
	1304	+ if (unmap_all) {
	1305	+ if (iova \|\| size)
	1306	+ goto unlock;
	1307	+ size = SIZE_MAX;
	1308	+ } else if (!size \|\| size & (pgsize - 1)) {
1092	1309	goto unlock;
1093	1310	}
1094	1311
1095		− if (unmap->iova + unmap->size - 1 < unmap->iova \|\|
1096		− unmap->size > SIZE_MAX) {
1097		− ret = -EINVAL;
	1312	+ if (iova + size - 1 < iova \|\| size > SIZE_MAX)
1098	1313	goto unlock;
1099		− }
1100	1314
1101	1315	/* When dirty tracking is enabled, allow only min supported pgsize */
1102	1316	if ((unmap->flags & VFIO_DMA_UNMAP_FLAG_GET_DIRTY_BITMAP) &&
1103	1317	(!iommu->dirty_page_tracking \|\| (bitmap->pgsize != pgsize))) {
1104		− ret = -EINVAL;
1105	1318	goto unlock;
1106	1319	}
1107	1320
@@ -1138,21 +1351,25 @@ again: · 30 unmodified lines
1138	1351	* will only return success and a size of zero if there were no
1139	1352	* mappings within the range.
1140	1353	*/
1141		− if (iommu->v2) {
1142		− dma = vfio_find_dma(iommu, unmap->iova, 1);
1143		− if (dma && dma->iova != unmap->iova) {
1144		− ret = -EINVAL;
	1354	+ if (iommu->v2 && !unmap_all) {
	1355	+ dma = vfio_find_dma(iommu, iova, 1);
	1356	+ if (dma && dma->iova != iova)
1145	1357	goto unlock;
1146		− }
1147		− dma = vfio_find_dma(iommu, unmap->iova + unmap->size - 1, 0);
1148		− if (dma && dma->iova + dma->size != unmap->iova + unmap->size) {
1149		− ret = -EINVAL;
	1358	+
	1359	+ dma = vfio_find_dma(iommu, iova + size - 1, 0);
	1360	+ if (dma && dma->iova + dma->size != iova + size)
1150	1361	goto unlock;
1151		− }
1152	1362	}
1153	1363
1154		− while ((dma = vfio_find_dma(iommu, unmap->iova, unmap->size))) {
1155		− if (!iommu->v2 && unmap->iova > dma->iova)
	1364	+ ret = 0;
	1365	+ n = first_n = vfio_find_dma_first_node(iommu, iova, size);
	1366	+
	1367	+ while (n) {
	1368	+ dma = rb_entry(n, struct vfio_dma, node);
	1369	+ if (dma->iova >= iova + size)
	1370	+ break;
	1371	+
	1372	+ if (!iommu->v2 && iova > dma->iova)
1156	1373	break;
1157	1374	/*
1158	1375	* Task with same address space who mapped this iova range is
@@ -1161,6 +1378,27 @@ again: · 2 unmodified lines
1161	1378	if (dma->task->mm != current->mm)
1162	1379	break;
1163	1380
	1381	+ if (invalidate_vaddr) {
	1382	+ if (dma->vaddr_invalid) {
	1383	+ struct rb_node *last_n = n;
	1384	+
	1385	+ for (n = first_n; n != last_n; n = rb_next(n)) {
	1386	+ dma = rb_entry(n,
	1387	+ struct vfio_dma, node);
	1388	+ dma->vaddr_invalid = false;
	1389	+ iommu->vaddr_invalid_count--;
	1390	+ }
	1391	+ ret = -EINVAL;
	1392	+ unmapped = 0;
	1393	+ break;
	1394	+ }
	1395	+ dma->vaddr_invalid = true;
	1396	+ iommu->vaddr_invalid_count++;
	1397	+ unmapped += dma->size;
	1398	+ n = rb_next(n);
	1399	+ continue;
	1400	+ }
	1401	+
1164	1402	if (!RB_EMPTY_ROOT(&dma->pfn_list)) {
1165	1403	struct vfio_iommu_type1_dma_unmap nb_unmap;
1166	1404
@@ -1190,12 +1428,13 @@ again: · 23 unmodified lines
1190	1428
1191	1429	if (unmap->flags & VFIO_DMA_UNMAP_FLAG_GET_DIRTY_BITMAP) {
1192	1430	ret = update_user_bitmap(bitmap->data, iommu, dma,
1193		− unmap->iova, pgsize);
	1431	+ iova, pgsize);
1194	1432	if (ret)
1195	1433	break;
1196	1434	}
1197	1435
1198	1436	unmapped += dma->size;
	1437	+ n = rb_next(n);
1199	1438	vfio_remove_dma(iommu, dma);
1200	1439	}
1201	1440
@@ -1239,15 +1478,19 @@ static int vfio_pin_map_dma(struct vfio_iommu iommu, struct vfio_dma dma, · 37 unmodified lines
1239	1478	{
1240	1479	dma_addr_t iova = dma->iova;
1241	1480	unsigned long vaddr = dma->vaddr;
	1481	+ struct vfio_batch batch;
1242	1482	size_t size = map_size;
1243	1483	long npage;
1244	1484	unsigned long pfn, limit = rlimit(RLIMIT_MEMLOCK) >> PAGE_SHIFT;
1245	1485	int ret = 0;
1246	1486
	1487	+ vfio_batch_init(&batch);
	1488	+
1247	1489	while (size) {
1248	1490	/* Pin a contiguous chunk of memory */
1249	1491	npage = vfio_pin_pages_remote(dma, vaddr + dma->size,
1250		− size >> PAGE_SHIFT, &pfn, limit);
	1492	+ size >> PAGE_SHIFT, &pfn, limit,
	1493	+ &batch);
1251	1494	if (npage <= 0) {
1252	1495	WARN_ON(!npage);
1253	1496	ret = (int)npage;
@@ -1260,6 +1503,7 @@ static int vfio_pin_map_dma(struct vfio_iommu iommu, struct vfio_dma dma, · 6 unmodified lines
1260	1503	if (ret) {
1261	1504	vfio_unpin_pages_remote(dma, iova + dma->size, pfn,
1262	1505	npage, true);
	1506	+ vfio_batch_unpin(&batch, dma);
1263	1507	break;
1264	1508	}
1265	1509
@@ -1267,6 +1511,7 @@ static int vfio_pin_map_dma(struct vfio_iommu iommu, struct vfio_dma dma, · 1 unmodified line
1267	1511	dma->size += npage << PAGE_SHIFT;
1268	1512	}
1269	1513
	1514	+ vfio_batch_fini(&batch);
1270	1515	dma->iommu_mapped = true;
1271	1516
1272	1517	if (ret)
@@ -1299,6 +1544,7 @@ static bool vfio_iommu_iova_dma_valid(struct vfio_iommu *iommu, · 26 unmodified lines
1299	1544	static int vfio_dma_do_map(struct vfio_iommu *iommu,
1300	1545	struct vfio_iommu_type1_dma_map *map)
1301	1546	{
	1547	+ bool set_vaddr = map->flags & VFIO_DMA_MAP_FLAG_VADDR;
1302	1548	dma_addr_t iova = map->iova;
1303	1549	unsigned long vaddr = map->vaddr;
1304	1550	size_t size = map->size;
@@ -1316,13 +1562,16 @@ static int vfio_dma_do_map(struct vfio_iommu *iommu, · 11 unmodified lines
1316	1562	if (map->flags & VFIO_DMA_MAP_FLAG_READ)
1317	1563	prot \|= IOMMU_READ;
1318	1564
	1565	+ if ((prot && set_vaddr) \|\| (!prot && !set_vaddr))
	1566	+ return -EINVAL;
	1567	+
1319	1568	mutex_lock(&iommu->lock);
1320	1569
1321	1570	pgsize = (size_t)1 << __ffs(iommu->pgsize_bitmap);
1322	1571
1323	1572	WARN_ON((pgsize - 1) & PAGE_MASK);
1324	1573
1325		− if (!prot \|\| !size \|\| (size \| iova \| vaddr) & (pgsize - 1)) {
	1574	+ if (!size \|\| (size \| iova \| vaddr) & (pgsize - 1)) {
1326	1575	ret = -EINVAL;
1327	1576	goto out_unlock;
1328	1577	}
@@ -1333,7 +1582,21 @@ static int vfio_dma_do_map(struct vfio_iommu *iommu, · 4 unmodified lines
1333	1582	goto out_unlock;
1334	1583	}
1335	1584
1336		− if (vfio_find_dma(iommu, iova, size)) {
	1585	+ dma = vfio_find_dma(iommu, iova, size);
	1586	+ if (set_vaddr) {
	1587	+ if (!dma) {
	1588	+ ret = -ENOENT;
	1589	+ } else if (!dma->vaddr_invalid \|\| dma->iova != iova \|\|
	1590	+ dma->size != size) {
	1591	+ ret = -EINVAL;
	1592	+ } else {
	1593	+ dma->vaddr = vaddr;
	1594	+ dma->vaddr_invalid = false;
	1595	+ iommu->vaddr_invalid_count--;
	1596	+ wake_up_all(&iommu->vaddr_wait);
	1597	+ }
	1598	+ goto out_unlock;
	1599	+ } else if (dma) {
1337	1600	ret = -EEXIST;
1338	1601	goto out_unlock;
1339	1602	}
@@ -1425,16 +1688,23 @@ static int vfio_bus_type(struct device dev, void data) · 85 unmodified lines
1425	1688	static int vfio_iommu_replay(struct vfio_iommu *iommu,
1426	1689	struct vfio_domain *domain)
1427	1690	{
	1691	+ struct vfio_batch batch;
1428	1692	struct vfio_domain *d = NULL;
1429	1693	struct rb_node *n;
1430	1694	unsigned long limit = rlimit(RLIMIT_MEMLOCK) >> PAGE_SHIFT;
1431	1695	int ret;
1432	1696
	1697	+ ret = vfio_wait_all_valid(iommu);
	1698	+ if (ret < 0)
	1699	+ return ret;
	1700	+
1433	1701	/* Arbitrarily pick the first domain in the list for lookups */
1434	1702	if (!list_empty(&iommu->domain_list))
1435	1703	d = list_first_entry(&iommu->domain_list,
1436	1704	struct vfio_domain, next);
1437	1705
	1706	+ vfio_batch_init(&batch);
	1707	+
1438	1708	n = rb_first(&iommu->dma_list);
1439	1709
1440	1710	for (; n; n = rb_next(n)) {
@@ -1482,7 +1752,8 @@ static int vfio_iommu_replay(struct vfio_iommu *iommu, · 41 unmodified lines
1482	1752
1483	1753	npage = vfio_pin_pages_remote(dma, vaddr,
1484	1754	n >> PAGE_SHIFT,
1485		− &pfn, limit);
	1755	+ &pfn, limit,
	1756	+ &batch);
1486	1757	if (npage <= 0) {
1487	1758	WARN_ON(!npage);
1488	1759	ret = (int)npage;
@@ -1496,11 +1767,13 @@ static int vfio_iommu_replay(struct vfio_iommu *iommu, · 7 unmodified lines
1496	1767	ret = iommu_map(domain->domain, iova, phys,
1497	1768	size, dma->prot \| domain->prot);
1498	1769	if (ret) {
1499		− if (!dma->iommu_mapped)
	1770	+ if (!dma->iommu_mapped) {
1500	1771	vfio_unpin_pages_remote(dma, iova,
1501	1772	phys >> PAGE_SHIFT,
1502	1773	size >> PAGE_SHIFT,
1503	1774	true);
	1775	+ vfio_batch_unpin(&batch, dma);
	1776	+ }
1504	1777	goto unwind;
1505	1778	}
1506	1779
@@ -1515,6 +1788,7 @@ static int vfio_iommu_replay(struct vfio_iommu *iommu, · 8 unmodified lines
1515	1788	dma->iommu_mapped = true;
1516	1789	}
1517	1790
	1791	+ vfio_batch_fini(&batch);
1518	1792	return 0;
1519	1793
1520	1794	unwind:
@@ -1555,6 +1829,7 @@ unwind: · 34 unmodified lines
1555	1829	}
1556	1830	}
1557	1831
	1832	+ vfio_batch_fini(&batch);
1558	1833	return ret;
1559	1834	}
1560	1835
@@ -1622,33 +1897,6 @@ static struct vfio_group vfio_iommu_find_iommu_group(struct vfio_iommu iommu, · 61 unmodified lines
1622	1897	return group;
1623	1898	}
1624	1899
1625		−static void update_pinned_page_dirty_scope(struct vfio_iommu *iommu)
1626		−{
1627		− struct vfio_domain *domain;
1628		− struct vfio_group *group;
1629		−
1630		− list_for_each_entry(domain, &iommu->domain_list, next) {
1631		− list_for_each_entry(group, &domain->group_list, next) {
1632		− if (!group->pinned_page_dirty_scope) {
1633		− iommu->pinned_page_dirty_scope = false;
1634		− return;
1635		− }
1636		− }
1637		− }
1638		−
1639		− if (iommu->external_domain) {
1640		− domain = iommu->external_domain;
1641		− list_for_each_entry(group, &domain->group_list, next) {
1642		− if (!group->pinned_page_dirty_scope) {
1643		− iommu->pinned_page_dirty_scope = false;
1644		− return;
1645		− }
1646		− }
1647		− }
1648		−
1649		− iommu->pinned_page_dirty_scope = true;
1650		−}
1651		−
1652	1900	static bool vfio_iommu_has_sw_msi(struct list_head *group_resv_regions,
1653	1901	phys_addr_t *base)
1654	1902	{
@@ -2057,8 +2305,6 @@ static int vfio_iommu_type1_attach_group(void *iommu_data, · 402 unmodified lines
2057	2305	* addition of a dirty tracking group.
2058	2306	*/
2059	2307	group->pinned_page_dirty_scope = true;
2060		− if (!iommu->pinned_page_dirty_scope)
2061		− update_pinned_page_dirty_scope(iommu);
2062	2308	mutex_unlock(&iommu->lock);
2063	2309
2064	2310	return 0;
@@ -2188,7 +2434,7 @@ done: · 123 unmodified lines
2188	2434	* demotes the iommu scope until it declares itself dirty tracking
2189	2435	* capable via the page pinning interface.
2190	2436	*/
2191		− iommu->pinned_page_dirty_scope = false;
	2437	+ iommu->num_non_pinned_groups++;
2192	2438	mutex_unlock(&iommu->lock);
2193	2439	vfio_iommu_resv_free(&group_resv_regions);
2194	2440
@@ -2238,23 +2484,6 @@ static void vfio_iommu_unmap_unpin_reaccount(struct vfio_iommu *iommu) · 43 unmodified lines
2238	2484	}
2239	2485	}
2240	2486
2241		−static void vfio_sanity_check_pfn_list(struct vfio_iommu *iommu)
2242		−{
2243		− struct rb_node *n;
2244		−
2245		− n = rb_first(&iommu->dma_list);
2246		− for (; n; n = rb_next(n)) {
2247		− struct vfio_dma *dma;
2248		−
2249		− dma = rb_entry(n, struct vfio_dma, node);
2250		−
2251		− if (WARN_ON(!RB_EMPTY_ROOT(&dma->pfn_list)))
2252		− break;
2253		− }
2254		− /* mdev vendor driver must unregister notifier */
2255		− WARN_ON(iommu->notifier.head);
2256		−}
2257		−
2258	2487	/*
2259	2488	* Called when a domain is removed in detach. It is possible that
2260	2489	* the removed domain decided the iova aperture window. Modify the
@@ -2354,10 +2583,10 @@ static void vfio_iommu_type1_detach_group(void *iommu_data, · 93 unmodified lines
2354	2583	kfree(group);
2355	2584
2356	2585	if (list_empty(&iommu->external_domain->group_list)) {
2357		− vfio_sanity_check_pfn_list(iommu);
2358		−
2359		− if (!IS_IOMMU_CAP_DOMAIN_IN_CONTAINER(iommu))
	2586	+ if (!IS_IOMMU_CAP_DOMAIN_IN_CONTAINER(iommu)) {
	2587	+ WARN_ON(iommu->notifier.head);
2360	2588	vfio_iommu_unmap_unpin_all(iommu);
	2589	+ }
2361	2590
2362	2591	kfree(iommu->external_domain);
2363	2592	iommu->external_domain = NULL;
@@ -2391,10 +2620,12 @@ static void vfio_iommu_type1_detach_group(void *iommu_data, · 27 unmodified lines
2391	2620	*/
2392	2621	if (list_empty(&domain->group_list)) {
2393	2622	if (list_is_singular(&iommu->domain_list)) {
2394		− if (!iommu->external_domain)
	2623	+ if (!iommu->external_domain) {
	2624	+ WARN_ON(iommu->notifier.head);
2395	2625	vfio_iommu_unmap_unpin_all(iommu);
2396		− else
	2626	+ } else {
2397	2627	vfio_iommu_unmap_unpin_reaccount(iommu);
	2628	+ }
2398	2629	}
2399	2630	iommu_domain_free(domain->domain);
2400	2631	list_del(&domain->next);
@@ -2415,8 +2646,11 @@ detach_group_done: · 14 unmodified lines
2415	2646	* Removal of a group without dirty tracking may allow the iommu scope
2416	2647	* to be promoted.
2417	2648	*/
2418		− if (update_dirty_scope)
2419		− update_pinned_page_dirty_scope(iommu);
	2649	+ if (update_dirty_scope) {
	2650	+ iommu->num_non_pinned_groups--;
	2651	+ if (iommu->dirty_page_tracking)
	2652	+ vfio_iommu_populate_bitmap_full(iommu);
	2653	+ }
2420	2654	mutex_unlock(&iommu->lock);
2421	2655	}
2422	2656
@@ -2446,8 +2680,10 @@ static void *vfio_iommu_type1_open(unsigned long arg) · 23 unmodified lines
2446	2680	INIT_LIST_HEAD(&iommu->iova_list);
2447	2681	iommu->dma_list = RB_ROOT;
2448	2682	iommu->dma_avail = dma_entry_limit;
	2683	+ iommu->container_open = true;
2449	2684	mutex_init(&iommu->lock);
2450	2685	BLOCKING_INIT_NOTIFIER_HEAD(&iommu->notifier);
	2686	+ init_waitqueue_head(&iommu->vaddr_wait);
2451	2687
2452	2688	return iommu;
2453	2689	}
@@ -2475,7 +2711,6 @@ static void vfio_iommu_type1_release(void *iommu_data) · 21 unmodified lines
2475	2711
2476	2712	if (iommu->external_domain) {
2477	2713	vfio_release_domain(iommu->external_domain, true);
2478		− vfio_sanity_check_pfn_list(iommu);
2479	2714	kfree(iommu->external_domain);
2480	2715	}
2481	2716
@@ -2517,6 +2752,8 @@ static int vfio_iommu_type1_check_extension(struct vfio_iommu *iommu, · 35 unmodified lines
2517	2752	case VFIO_TYPE1_IOMMU:
2518	2753	case VFIO_TYPE1v2_IOMMU:
2519	2754	case VFIO_TYPE1_NESTING_IOMMU:
	2755	+ case VFIO_UNMAP_ALL:
	2756	+ case VFIO_UPDATE_VADDR:
2520	2757	return 1;
2521	2758	case VFIO_DMA_CC_IOMMU:
2522	2759	if (!iommu)
@@ -2688,7 +2925,8 @@ static int vfio_iommu_type1_map_dma(struct vfio_iommu *iommu, · 165 unmodified lines
2688	2925	{
2689	2926	struct vfio_iommu_type1_dma_map map;
2690	2927	unsigned long minsz;
2691		− uint32_t mask = VFIO_DMA_MAP_FLAG_READ \| VFIO_DMA_MAP_FLAG_WRITE;
	2928	+ uint32_t mask = VFIO_DMA_MAP_FLAG_READ \| VFIO_DMA_MAP_FLAG_WRITE \|
	2929	+ VFIO_DMA_MAP_FLAG_VADDR;
2692	2930
2693	2931	minsz = offsetofend(struct vfio_iommu_type1_dma_map, size);
2694	2932
@@ -2706,6 +2944,9 @@ static int vfio_iommu_type1_unmap_dma(struct vfio_iommu *iommu, · 11 unmodified lines
2706	2944	{
2707	2945	struct vfio_iommu_type1_dma_unmap unmap;
2708	2946	struct vfio_bitmap bitmap = { 0 };
	2947	+ uint32_t mask = VFIO_DMA_UNMAP_FLAG_GET_DIRTY_BITMAP \|
	2948	+ VFIO_DMA_UNMAP_FLAG_VADDR \|
	2949	+ VFIO_DMA_UNMAP_FLAG_ALL;
2709	2950	unsigned long minsz;
2710	2951	int ret;
2711	2952
@@ -2714,8 +2955,12 @@ static int vfio_iommu_type1_unmap_dma(struct vfio_iommu *iommu, · 2 unmodified lines
2714	2955	if (copy_from_user(&unmap, (void __user *)arg, minsz))
2715	2956	return -EFAULT;
2716	2957
2717		− if (unmap.argsz < minsz \|\|
2718		− unmap.flags & ~VFIO_DMA_UNMAP_FLAG_GET_DIRTY_BITMAP)
	2958	+ if (unmap.argsz < minsz \|\| unmap.flags & ~mask)
	2959	+ return -EINVAL;
	2960	+
	2961	+ if ((unmap.flags & VFIO_DMA_UNMAP_FLAG_GET_DIRTY_BITMAP) &&
	2962	+ (unmap.flags & (VFIO_DMA_UNMAP_FLAG_ALL \|
	2963	+ VFIO_DMA_UNMAP_FLAG_VADDR)))
2719	2964	return -EINVAL;
2720	2965
2721	2966	if (unmap.flags & VFIO_DMA_UNMAP_FLAG_GET_DIRTY_BITMAP) {
@@ -2906,12 +3151,13 @@ static int vfio_iommu_type1_dma_rw_chunk(struct vfio_iommu *iommu, · 184 unmodified lines
2906	3151	struct vfio_dma *dma;
2907	3152	bool kthread = current->mm == NULL;
2908	3153	size_t offset;
	3154	+ int ret;
2909	3155
2910	3156	*copied = 0;
2911	3157
2912		− dma = vfio_find_dma(iommu, user_iova, 1);
2913		− if (!dma)
2914		− return -EINVAL;
	3158	+ ret = vfio_find_dma_valid(iommu, user_iova, 1, &dma);
	3159	+ if (ret < 0)
	3160	+ return ret;
2915	3161
2916	3162	if ((write && !(dma->prot & IOMMU_WRITE)) \|\|
2917	3163	!(dma->prot & IOMMU_READ))
@@ -3003,6 +3249,19 @@ vfio_iommu_type1_group_iommu_domain(void *iommu_data, · 85 unmodified lines
3003	3249	return domain;
3004	3250	}
3005	3251
	3252	+static void vfio_iommu_type1_notify(void *iommu_data,
	3253	+ enum vfio_iommu_notify_type event)
	3254	+{
	3255	+ struct vfio_iommu *iommu = iommu_data;
	3256	+
	3257	+ if (event != VFIO_IOMMU_CONTAINER_CLOSE)
	3258	+ return;
	3259	+ mutex_lock(&iommu->lock);
	3260	+ iommu->container_open = false;
	3261	+ mutex_unlock(&iommu->lock);
	3262	+ wake_up_all(&iommu->vaddr_wait);
	3263	+}
	3264	+
3006	3265	static const struct vfio_iommu_driver_ops vfio_iommu_driver_ops_type1 = {
3007	3266	.name = "vfio-iommu-type1",
3008	3267	.owner = THIS_MODULE,
@@ -3017,6 +3276,7 @@ static const struct vfio_iommu_driver_ops vfio_iommu_driver_ops_type1 = { · 8 unmodified lines
3017	3276	.unregister_notifier = vfio_iommu_type1_unregister_notifier,
3018	3277	.dma_rw = vfio_iommu_type1_dma_rw,
3019	3278	.group_iommu_domain = vfio_iommu_type1_group_iommu_domain,
	3279	+ .notify = vfio_iommu_type1_notify,
3020	3280	};
3021	3281
3022	3282	static int __init vfio_iommu_type1_init(void)

include/linux/vfio.h+7−0

@@ -57,6 +57,11 @@ extern struct vfio_device vfio_device_get_from_dev(struct device dev); · 56 unmodified lines
57	57	extern void vfio_device_put(struct vfio_device *device);
58	58	extern void vfio_device_data(struct vfio_device device);
59	59
	60	+/* events for the backend driver notify callback */
	61	+enum vfio_iommu_notify_type {
	62	+ VFIO_IOMMU_CONTAINER_CLOSE = 0,
	63	+};
	64	+
60	65	/**
61	66	* struct vfio_iommu_driver_ops - VFIO IOMMU driver callbacks
62	67	*/
@@ -92,6 +97,8 @@ struct vfio_iommu_driver_ops { · 29 unmodified lines
92	97	void *data, size_t count, bool write);
93	98	struct iommu_domain (group_iommu_domain)(void *iommu_data,
94	99	struct iommu_group *group);
	100	+ void (notify)(void iommu_data,
	101	+ enum vfio_iommu_notify_type event);
95	102	};
96	103
97	104	extern int vfio_register_iommu_driver(const struct vfio_iommu_driver_ops *ops);

include/uapi/linux/vfio.h+27−0

@@ -46,6 +46,12 @@ · 45 unmodified lines
46	46	*/
47	47	#define VFIO_NOIOMMU_IOMMU 8
48	48
	49	+/* Supports VFIO_DMA_UNMAP_FLAG_ALL */
	50	+#define VFIO_UNMAP_ALL 9
	51	+
	52	+/* Supports the vaddr flag for DMA map and unmap */
	53	+#define VFIO_UPDATE_VADDR 10
	54	+
49	55	/*
50	56	* The IOCTL interface is designed for extensibility by embedding the
51	57	* structure length (argsz) and flags into structures passed between
@@ -1074,12 +1080,22 @@ struct vfio_iommu_type1_info_dma_avail { · 1022 unmodified lines
1074	1080	*
1075	1081	* Map process virtual addresses to IO virtual addresses using the
1076	1082	* provided struct vfio_dma_map. Caller sets argsz. READ &/ WRITE required.
	1083	+ *
	1084	+ * If flags & VFIO_DMA_MAP_FLAG_VADDR, update the base vaddr for iova, and
	1085	+ * unblock translation of host virtual addresses in the iova range. The vaddr
	1086	+ * must have previously been invalidated with VFIO_DMA_UNMAP_FLAG_VADDR. To
	1087	+ * maintain memory consistency within the user application, the updated vaddr
	1088	+ * must address the same memory object as originally mapped. Failure to do so
	1089	+ * will result in user memory corruption and/or device misbehavior. iova and
	1090	+ * size must match those in the original MAP_DMA call. Protection is not
	1091	+ * changed, and the READ & WRITE flags must be 0.
1077	1092	*/
1078	1093	struct vfio_iommu_type1_dma_map {
1079	1094	__u32 argsz;
1080	1095	__u32 flags;
1081	1096	#define VFIO_DMA_MAP_FLAG_READ (1 << 0) /* readable from device */
1082	1097	#define VFIO_DMA_MAP_FLAG_WRITE (1 << 1) /* writable from device */
	1098	+#define VFIO_DMA_MAP_FLAG_VADDR (1 << 2)
1083	1099	__u64 vaddr; /* Process virtual address */
1084	1100	__u64 iova; /* IO virtual address */
1085	1101	__u64 size; /* Size of mapping (bytes) */
@@ -1102,6 +1118,7 @@ struct vfio_bitmap { · 16 unmodified lines
1102	1118	* field. No guarantee is made to the user that arbitrary unmaps of iova
1103	1119	* or size different from those used in the original mapping call will
1104	1120	* succeed.
	1121	+ *
1105	1122	* VFIO_DMA_UNMAP_FLAG_GET_DIRTY_BITMAP should be set to get the dirty bitmap
1106	1123	* before unmapping IO virtual addresses. When this flag is set, the user must
1107	1124	* provide a struct vfio_bitmap in data[]. User must provide zero-allocated
@@ -1111,11 +1128,21 @@ struct vfio_bitmap { · 3 unmodified lines
1111	1128	* indicates that the page at that offset from iova is dirty. A Bitmap of the
1112	1129	* pages in the range of unmapped size is returned in the user-provided
1113	1130	* vfio_bitmap.data.
	1131	+ *
	1132	+ * If flags & VFIO_DMA_UNMAP_FLAG_ALL, unmap all addresses. iova and size
	1133	+ * must be 0. This cannot be combined with the get-dirty-bitmap flag.
	1134	+ *
	1135	+ * If flags & VFIO_DMA_UNMAP_FLAG_VADDR, do not unmap, but invalidate host
	1136	+ * virtual addresses in the iova range. Tasks that attempt to translate an
	1137	+ * iova's vaddr will block. DMA to already-mapped pages continues. This
	1138	+ * cannot be combined with the get-dirty-bitmap flag.
1114	1139	*/
1115	1140	struct vfio_iommu_type1_dma_unmap {
1116	1141	__u32 argsz;
1117	1142	__u32 flags;
1118	1143	#define VFIO_DMA_UNMAP_FLAG_GET_DIRTY_BITMAP (1 << 0)
	1144	+#define VFIO_DMA_UNMAP_FLAG_ALL (1 << 1)
	1145	+#define VFIO_DMA_UNMAP_FLAG_VADDR (1 << 2)
1119	1146	__u64 iova; /* IO virtual address */
1120	1147	__u64 size; /* Size of mapping (bytes) */
1121	1148	__u8 data[];