vhost: factor out IOTLB

This patch factors out IOTLB into a dedicated module in order to be reused by other modules like vringh. User may choose to enable the automatic retiring by specifying VHOST_IOTLB_FLAG_RETIRE flag to fit for the case of vhost device IOTLB implementation.

Jason Wang committed 6 years ago

Commit 0bbe3061 parent 792a4f28 files · +315 −181Browse files

Unified · Split

8 files

MAINTAINERS +1 −0
drivers/vhost/Kconfig +6 −0
drivers/vhost/Makefile +3 −0
drivers/vhost/iotlb.c +177 −0
drivers/vhost/net.c +1 −1
drivers/vhost/vhost.c +69 −152
drivers/vhost/vhost.h +11 −28
include/linux/vhost_iotlb.h +47 −0

MAINTAINERS+1−0

@@ -17766,6 +17766,7 @@ T: git git://git.kernel.org/pub/scm/linux/kernel/git/mst/vhost.git · 17765 unmodified lines
17766	17766	S: Maintained
17767	17767	F: drivers/vhost/
17768	17768	F: include/uapi/linux/vhost.h
	17769	+F: include/linux/vhost_iotlb.h
17769	17770
17770	17771	VIRTIO INPUT DRIVER
17771	17772	M: Gerd Hoffmann <kraxel@redhat.com>

drivers/vhost/Kconfig+6−0

@@ -1,4 +1,9 @@
1	1	# SPDX-License-Identifier: GPL-2.0-only
	2	+config VHOST_IOTLB
	3	+ tristate
	4	+ help
	5	+ Generic IOTLB implementation for vhost and vringh.
	6	+
2	7	config VHOST_RING
3	8	tristate
4	9	help
@@ -67,4 +72,5 @@ config VHOST_CROSS_ENDIAN_LEGACY · 62 unmodified lines
67	72	adds some overhead, it is disabled by default.
68	73
69	74	If unsure, say "N".
	75	+
70	76	endif

drivers/vhost/Makefile+3−0

@@ -11,3 +11,6 @@ vhost_vsock-y := vsock.o · 10 unmodified lines
11	11	obj-$(CONFIG_VHOST_RING) += vringh.o
12	12
13	13	obj-$(CONFIG_VHOST) += vhost.o
	14	+
	15	+obj-$(CONFIG_VHOST_IOTLB) += vhost_iotlb.o
	16	+vhost_iotlb-y := iotlb.o

drivers/vhost/iotlb.c+177−0

@@ -0,0 +1,177 @@
	1	+// SPDX-License-Identifier: GPL-2.0-only
	2	+/* Copyright (C) 2020 Red Hat, Inc.
	3	+ * Author: Jason Wang <jasowang@redhat.com>
	4	+ *
	5	+ * IOTLB implementation for vhost.
	6	+ */
	7	+#include <linux/slab.h>
	8	+#include <linux/vhost_iotlb.h>
	9	+#include <linux/module.h>
	10	+
	11	+#define MOD_VERSION "0.1"
	12	+#define MOD_DESC "VHOST IOTLB"
	13	+#define MOD_AUTHOR "Jason Wang <jasowang@redhat.com>"
	14	+#define MOD_LICENSE "GPL v2"
	15	+
	16	+#define START(map) ((map)->start)
	17	+#define LAST(map) ((map)->last)
	18	+
	19	+INTERVAL_TREE_DEFINE(struct vhost_iotlb_map,
	20	+ rb, __u64, __subtree_last,
	21	+ START, LAST, static inline, vhost_iotlb_itree);
	22	+
	23	+/**
	24	+ * vhost_iotlb_map_free - remove a map node and free it
	25	+ * @iotlb: the IOTLB
	26	+ * @map: the map that want to be remove and freed
	27	+ */
	28	+void vhost_iotlb_map_free(struct vhost_iotlb *iotlb,
	29	+ struct vhost_iotlb_map *map)
	30	+{
	31	+ vhost_iotlb_itree_remove(map, &iotlb->root);
	32	+ list_del(&map->link);
	33	+ kfree(map);
	34	+ iotlb->nmaps--;
	35	+}
	36	+EXPORT_SYMBOL_GPL(vhost_iotlb_map_free);
	37	+
	38	+/**
	39	+ * vhost_iotlb_add_range - add a new range to vhost IOTLB
	40	+ * @iotlb: the IOTLB
	41	+ * @start: start of the IOVA range
	42	+ * @last: last of IOVA range
	43	+ * @addr: the address that is mapped to @start
	44	+ * @perm: access permission of this range
	45	+ *
	46	+ * Returns an error last is smaller than start or memory allocation
	47	+ * fails
	48	+ */
	49	+int vhost_iotlb_add_range(struct vhost_iotlb *iotlb,
	50	+ u64 start, u64 last,
	51	+ u64 addr, unsigned int perm)
	52	+{
	53	+ struct vhost_iotlb_map *map;
	54	+
	55	+ if (last < start)
	56	+ return -EFAULT;
	57	+
	58	+ if (iotlb->limit &&
	59	+ iotlb->nmaps == iotlb->limit &&
	60	+ iotlb->flags & VHOST_IOTLB_FLAG_RETIRE) {
	61	+ map = list_first_entry(&iotlb->list, typeof(*map), link);
	62	+ vhost_iotlb_map_free(iotlb, map);
	63	+ }
	64	+
	65	+ map = kmalloc(sizeof(*map), GFP_ATOMIC);
	66	+ if (!map)
	67	+ return -ENOMEM;
	68	+
	69	+ map->start = start;
	70	+ map->size = last - start + 1;
	71	+ map->last = last;
	72	+ map->addr = addr;
	73	+ map->perm = perm;
	74	+
	75	+ iotlb->nmaps++;
	76	+ vhost_iotlb_itree_insert(map, &iotlb->root);
	77	+
	78	+ INIT_LIST_HEAD(&map->link);
	79	+ list_add_tail(&map->link, &iotlb->list);
	80	+
	81	+ return 0;
	82	+}
	83	+EXPORT_SYMBOL_GPL(vhost_iotlb_add_range);
	84	+
	85	+/**
	86	+ * vring_iotlb_del_range - delete overlapped ranges from vhost IOTLB
	87	+ * @iotlb: the IOTLB
	88	+ * @start: start of the IOVA range
	89	+ * @last: last of IOVA range
	90	+ */
	91	+void vhost_iotlb_del_range(struct vhost_iotlb *iotlb, u64 start, u64 last)
	92	+{
	93	+ struct vhost_iotlb_map *map;
	94	+
	95	+ while ((map = vhost_iotlb_itree_iter_first(&iotlb->root,
	96	+ start, last)))
	97	+ vhost_iotlb_map_free(iotlb, map);
	98	+}
	99	+EXPORT_SYMBOL_GPL(vhost_iotlb_del_range);
	100	+
	101	+/**
	102	+ * vhost_iotlb_alloc - add a new vhost IOTLB
	103	+ * @limit: maximum number of IOTLB entries
	104	+ * @flags: VHOST_IOTLB_FLAG_XXX
	105	+ *
	106	+ * Returns an error is memory allocation fails
	107	+ */
	108	+struct vhost_iotlb *vhost_iotlb_alloc(unsigned int limit, unsigned int flags)
	109	+{
	110	+ struct vhost_iotlb iotlb = kzalloc(sizeof(iotlb), GFP_KERNEL);
	111	+
	112	+ if (!iotlb)
	113	+ return NULL;
	114	+
	115	+ iotlb->root = RB_ROOT_CACHED;
	116	+ iotlb->limit = limit;
	117	+ iotlb->nmaps = 0;
	118	+ iotlb->flags = flags;
	119	+ INIT_LIST_HEAD(&iotlb->list);
	120	+
	121	+ return iotlb;
	122	+}
	123	+EXPORT_SYMBOL_GPL(vhost_iotlb_alloc);
	124	+
	125	+/**
	126	+ * vhost_iotlb_reset - reset vhost IOTLB (free all IOTLB entries)
	127	+ * @iotlb: the IOTLB to be reset
	128	+ */
	129	+void vhost_iotlb_reset(struct vhost_iotlb *iotlb)
	130	+{
	131	+ vhost_iotlb_del_range(iotlb, 0ULL, 0ULL - 1);
	132	+}
	133	+EXPORT_SYMBOL_GPL(vhost_iotlb_reset);
	134	+
	135	+/**
	136	+ * vhost_iotlb_free - reset and free vhost IOTLB
	137	+ * @iotlb: the IOTLB to be freed
	138	+ */
	139	+void vhost_iotlb_free(struct vhost_iotlb *iotlb)
	140	+{
	141	+ if (iotlb) {
	142	+ vhost_iotlb_reset(iotlb);
	143	+ kfree(iotlb);
	144	+ }
	145	+}
	146	+EXPORT_SYMBOL_GPL(vhost_iotlb_free);
	147	+
	148	+/**
	149	+ * vhost_iotlb_itree_first - return the first overlapped range
	150	+ * @iotlb: the IOTLB
	151	+ * @start: start of IOVA range
	152	+ * @end: end of IOVA range
	153	+ */
	154	+struct vhost_iotlb_map *
	155	+vhost_iotlb_itree_first(struct vhost_iotlb *iotlb, u64 start, u64 last)
	156	+{
	157	+ return vhost_iotlb_itree_iter_first(&iotlb->root, start, last);
	158	+}
	159	+EXPORT_SYMBOL_GPL(vhost_iotlb_itree_first);
	160	+
	161	+/**
	162	+ * vhost_iotlb_itree_first - return the next overlapped range
	163	+ * @iotlb: the IOTLB
	164	+ * @start: start of IOVA range
	165	+ * @end: end of IOVA range
	166	+ */
	167	+struct vhost_iotlb_map *
	168	+vhost_iotlb_itree_next(struct vhost_iotlb_map *map, u64 start, u64 last)
	169	+{
	170	+ return vhost_iotlb_itree_iter_next(map, start, last);
	171	+}
	172	+EXPORT_SYMBOL_GPL(vhost_iotlb_itree_next);
	173	+
	174	+MODULE_VERSION(MOD_VERSION);
	175	+MODULE_DESCRIPTION(MOD_DESC);
	176	+MODULE_AUTHOR(MOD_AUTHOR);
	177	+MODULE_LICENSE(MOD_LICENSE);

drivers/vhost/net.c+1−1

@@ -1587,7 +1587,7 @@ static long vhost_net_reset_owner(struct vhost_net *n) · 1586 unmodified lines
1587	1587	struct socket *tx_sock = NULL;
1588	1588	struct socket *rx_sock = NULL;
1589	1589	long err;
1590		− struct vhost_umem *umem;
	1590	+ struct vhost_iotlb *umem;
1591	1591
1592	1592	mutex_lock(&n->dev.mutex);
1593	1593	err = vhost_dev_check_owner(&n->dev);

drivers/vhost/vhost.c+69−152

@@ -50,10 +50,6 @@ enum { · 49 unmodified lines
50	50	#define vhost_used_event(vq) ((__virtio16 __user *)&vq->avail->ring[vq->num])
51	51	#define vhost_avail_event(vq) ((__virtio16 __user *)&vq->used->ring[vq->num])
52	52
53		−INTERVAL_TREE_DEFINE(struct vhost_umem_node,
54		− rb, __u64, __subtree_last,
55		− START, LAST, static inline, vhost_umem_interval_tree);
56		−
57	53	#ifdef CONFIG_VHOST_CROSS_ENDIAN_LEGACY
58	54	static void vhost_disable_cross_endian(struct vhost_virtqueue *vq)
59	55	{
@@ -584,21 +580,25 @@ err_mm: · 524 unmodified lines
584	580	}
585	581	EXPORT_SYMBOL_GPL(vhost_dev_set_owner);
586	582
587		−struct vhost_umem *vhost_dev_reset_owner_prepare(void)
	583	+static struct vhost_iotlb *iotlb_alloc(void)
	584	+{
	585	+ return vhost_iotlb_alloc(max_iotlb_entries,
	586	+ VHOST_IOTLB_FLAG_RETIRE);
	587	+}
	588	+
	589	+struct vhost_iotlb *vhost_dev_reset_owner_prepare(void)
588	590	{
589		− return kvzalloc(sizeof(struct vhost_umem), GFP_KERNEL);
	591	+ return iotlb_alloc();
590	592	}
591	593	EXPORT_SYMBOL_GPL(vhost_dev_reset_owner_prepare);
592	594
593	595	/* Caller should have device mutex */
594		−void vhost_dev_reset_owner(struct vhost_dev dev, struct vhost_umem umem)
	596	+void vhost_dev_reset_owner(struct vhost_dev dev, struct vhost_iotlb umem)
595	597	{
596	598	int i;
597	599
598	600	vhost_dev_cleanup(dev);
599	601
600		− /* Restore memory to default empty mapping. */
601		− INIT_LIST_HEAD(&umem->umem_list);
602	602	dev->umem = umem;
603	603	/* We don't need VQ locks below since vhost_dev_cleanup makes sure
604	604	* VQs aren't running.
@@ -621,28 +621,6 @@ void vhost_dev_stop(struct vhost_dev *dev) · 16 unmodified lines
621	621	}
622	622	EXPORT_SYMBOL_GPL(vhost_dev_stop);
623	623
624		−static void vhost_umem_free(struct vhost_umem *umem,
625		− struct vhost_umem_node *node)
626		−{
627		− vhost_umem_interval_tree_remove(node, &umem->umem_tree);
628		− list_del(&node->link);
629		− kfree(node);
630		− umem->numem--;
631		−}
632		−
633		−static void vhost_umem_clean(struct vhost_umem *umem)
634		−{
635		− struct vhost_umem_node node, tmp;
636		−
637		− if (!umem)
638		− return;
639		−
640		− list_for_each_entry_safe(node, tmp, &umem->umem_list, link)
641		− vhost_umem_free(umem, node);
642		−
643		− kvfree(umem);
644		−}
645		−
646	624	static void vhost_clear_msg(struct vhost_dev *dev)
647	625	{
648	626	struct vhost_msg_node node, n;
@@ -680,9 +658,9 @@ void vhost_dev_cleanup(struct vhost_dev *dev) · 31 unmodified lines
680	658	eventfd_ctx_put(dev->log_ctx);
681	659	dev->log_ctx = NULL;
682	660	/* No one will access memory at this point */
683		− vhost_umem_clean(dev->umem);
	661	+ vhost_iotlb_free(dev->umem);
684	662	dev->umem = NULL;
685		− vhost_umem_clean(dev->iotlb);
	663	+ vhost_iotlb_free(dev->iotlb);
686	664	dev->iotlb = NULL;
687	665	vhost_clear_msg(dev);
688	666	wake_up_interruptible_poll(&dev->wait, EPOLLIN \| EPOLLRDNORM);
@@ -718,27 +696,26 @@ static bool vhost_overflow(u64 uaddr, u64 size) · 29 unmodified lines
718	696	}
719	697
720	698	/* Caller should have vq mutex and device mutex. */
721		−static bool vq_memory_access_ok(void __user log_base, struct vhost_umem umem,
	699	+static bool vq_memory_access_ok(void __user log_base, struct vhost_iotlb umem,
722	700	int log_all)
723	701	{
724		− struct vhost_umem_node *node;
	702	+ struct vhost_iotlb_map *map;
725	703
726	704	if (!umem)
727	705	return false;
728	706
729		− list_for_each_entry(node, &umem->umem_list, link) {
730		− unsigned long a = node->userspace_addr;
	707	+ list_for_each_entry(map, &umem->list, link) {
	708	+ unsigned long a = map->addr;
731	709
732		− if (vhost_overflow(node->userspace_addr, node->size))
	710	+ if (vhost_overflow(map->addr, map->size))
733	711	return false;
734	712
735	713
736		− if (!access_ok((void __user *)a,
737		− node->size))
	714	+ if (!access_ok((void __user *)a, map->size))
738	715	return false;
739	716	else if (log_all && !log_access_ok(log_base,
740		− node->start,
741		− node->size))
	717	+ map->start,
	718	+ map->size))
742	719	return false;
743	720	}
744	721	return true;
@@ -748,17 +725,17 @@ static inline void __user vhost_vq_meta_fetch(struct vhost_virtqueue vq, · 3 unmodified lines
748	725	u64 addr, unsigned int size,
749	726	int type)
750	727	{
751		− const struct vhost_umem_node *node = vq->meta_iotlb[type];
	728	+ const struct vhost_iotlb_map *map = vq->meta_iotlb[type];
752	729
753		− if (!node)
	730	+ if (!map)
754	731	return NULL;
755	732
756		− return (void *)(uintptr_t)(node->userspace_addr + addr - node->start);
	733	+ return (void *)(uintptr_t)(map->addr + addr - map->start);
757	734	}
758	735
759	736	/* Can we switch to this memory table? */
760	737	/* Caller should have device mutex but not vq mutex */
761		−static bool memory_access_ok(struct vhost_dev d, struct vhost_umem umem,
	738	+static bool memory_access_ok(struct vhost_dev d, struct vhost_iotlb umem,
762	739	int log_all)
763	740	{
764	741	int i;
@@ -1023,47 +1000,6 @@ static inline int vhost_get_desc(struct vhost_virtqueue *vq, · 258 unmodified lines
1023	1000	return vhost_copy_from_user(vq, desc, vq->desc + idx, sizeof(*desc));
1024	1001	}
1025	1002
1026		−static int vhost_new_umem_range(struct vhost_umem *umem,
1027		− u64 start, u64 size, u64 end,
1028		− u64 userspace_addr, int perm)
1029		−{
1030		− struct vhost_umem_node tmp, node;
1031		−
1032		− if (!size)
1033		− return -EFAULT;
1034		−
1035		− node = kmalloc(sizeof(*node), GFP_ATOMIC);
1036		− if (!node)
1037		− return -ENOMEM;
1038		−
1039		− if (umem->numem == max_iotlb_entries) {
1040		− tmp = list_first_entry(&umem->umem_list, typeof(*tmp), link);
1041		− vhost_umem_free(umem, tmp);
1042		− }
1043		−
1044		− node->start = start;
1045		− node->size = size;
1046		− node->last = end;
1047		− node->userspace_addr = userspace_addr;
1048		− node->perm = perm;
1049		− INIT_LIST_HEAD(&node->link);
1050		− list_add_tail(&node->link, &umem->umem_list);
1051		− vhost_umem_interval_tree_insert(node, &umem->umem_tree);
1052		− umem->numem++;
1053		−
1054		− return 0;
1055		−}
1056		−
1057		−static void vhost_del_umem_range(struct vhost_umem *umem,
1058		− u64 start, u64 end)
1059		−{
1060		− struct vhost_umem_node *node;
1061		−
1062		− while ((node = vhost_umem_interval_tree_iter_first(&umem->umem_tree,
1063		− start, end)))
1064		− vhost_umem_free(umem, node);
1065		−}
1066		−
1067	1003	static void vhost_iotlb_notify_vq(struct vhost_dev *d,
1068	1004	struct vhost_iotlb_msg *msg)
1069	1005	{
@@ -1120,9 +1056,9 @@ static int vhost_process_iotlb_msg(struct vhost_dev *dev, · 50 unmodified lines
1120	1056	break;
1121	1057	}
1122	1058	vhost_vq_meta_reset(dev);
1123		− if (vhost_new_umem_range(dev->iotlb, msg->iova, msg->size,
1124		− msg->iova + msg->size - 1,
1125		− msg->uaddr, msg->perm)) {
	1059	+ if (vhost_iotlb_add_range(dev->iotlb, msg->iova,
	1060	+ msg->iova + msg->size - 1,
	1061	+ msg->uaddr, msg->perm)) {
1126	1062	ret = -ENOMEM;
1127	1063	break;
1128	1064	}
@@ -1134,8 +1070,8 @@ static int vhost_process_iotlb_msg(struct vhost_dev *dev, · 5 unmodified lines
1134	1070	break;
1135	1071	}
1136	1072	vhost_vq_meta_reset(dev);
1137		− vhost_del_umem_range(dev->iotlb, msg->iova,
1138		− msg->iova + msg->size - 1);
	1073	+ vhost_iotlb_del_range(dev->iotlb, msg->iova,
	1074	+ msg->iova + msg->size - 1);
1139	1075	break;
1140	1076	default:
1141	1077	ret = -EINVAL;
@@ -1319,44 +1255,42 @@ static bool vq_access_ok(struct vhost_virtqueue *vq, unsigned int num, · 177 unmodified lines
1319	1255	}
1320	1256
1321	1257	static void vhost_vq_meta_update(struct vhost_virtqueue *vq,
1322		− const struct vhost_umem_node *node,
	1258	+ const struct vhost_iotlb_map *map,
1323	1259	int type)
1324	1260	{
1325	1261	int access = (type == VHOST_ADDR_USED) ?
1326	1262	VHOST_ACCESS_WO : VHOST_ACCESS_RO;
1327	1263
1328		− if (likely(node->perm & access))
1329		− vq->meta_iotlb[type] = node;
	1264	+ if (likely(map->perm & access))
	1265	+ vq->meta_iotlb[type] = map;
1330	1266	}
1331	1267
1332	1268	static bool iotlb_access_ok(struct vhost_virtqueue *vq,
1333	1269	int access, u64 addr, u64 len, int type)
1334	1270	{
1335		− const struct vhost_umem_node *node;
1336		− struct vhost_umem *umem = vq->iotlb;
	1271	+ const struct vhost_iotlb_map *map;
	1272	+ struct vhost_iotlb *umem = vq->iotlb;
1337	1273	u64 s = 0, size, orig_addr = addr, last = addr + len - 1;
1338	1274
1339	1275	if (vhost_vq_meta_fetch(vq, addr, len, type))
1340	1276	return true;
1341	1277
1342	1278	while (len > s) {
1343		− node = vhost_umem_interval_tree_iter_first(&umem->umem_tree,
1344		− addr,
1345		− last);
1346		− if (node == NULL \|\| node->start > addr) {
	1279	+ map = vhost_iotlb_itree_first(umem, addr, last);
	1280	+ if (map == NULL \|\| map->start > addr) {
1347	1281	vhost_iotlb_miss(vq, addr, access);
1348	1282	return false;
1349		− } else if (!(node->perm & access)) {
	1283	+ } else if (!(map->perm & access)) {
1350	1284	/* Report the possible access violation by
1351	1285	* request another translation from userspace.
1352	1286	*/
1353	1287	return false;
1354	1288	}
1355	1289
1356		− size = node->size - addr + node->start;
	1290	+ size = map->size - addr + map->start;
1357	1291
1358	1292	if (orig_addr == addr && size >= len)
1359		− vhost_vq_meta_update(vq, node, type);
	1293	+ vhost_vq_meta_update(vq, map, type);
1360	1294
1361	1295	s += size;
1362	1296	addr += size;
@@ -1372,12 +1306,12 @@ int vq_meta_prefetch(struct vhost_virtqueue *vq) · 9 unmodified lines
1372	1306	if (!vq->iotlb)
1373	1307	return 1;
1374	1308
1375		− return iotlb_access_ok(vq, VHOST_ACCESS_RO, (u64)(uintptr_t)vq->desc,
	1309	+ return iotlb_access_ok(vq, VHOST_MAP_RO, (u64)(uintptr_t)vq->desc,
1376	1310	vhost_get_desc_size(vq, num), VHOST_ADDR_DESC) &&
1377		− iotlb_access_ok(vq, VHOST_ACCESS_RO, (u64)(uintptr_t)vq->avail,
	1311	+ iotlb_access_ok(vq, VHOST_MAP_RO, (u64)(uintptr_t)vq->avail,
1378	1312	vhost_get_avail_size(vq, num),
1379	1313	VHOST_ADDR_AVAIL) &&
1380		− iotlb_access_ok(vq, VHOST_ACCESS_WO, (u64)(uintptr_t)vq->used,
	1314	+ iotlb_access_ok(vq, VHOST_MAP_WO, (u64)(uintptr_t)vq->used,
1381	1315	vhost_get_used_size(vq, num), VHOST_ADDR_USED);
1382	1316	}
1383	1317	EXPORT_SYMBOL_GPL(vq_meta_prefetch);
@@ -1416,25 +1350,11 @@ bool vhost_vq_access_ok(struct vhost_virtqueue *vq) · 32 unmodified lines
1416	1350	}
1417	1351	EXPORT_SYMBOL_GPL(vhost_vq_access_ok);
1418	1352
1419		−static struct vhost_umem *vhost_umem_alloc(void)
1420		−{
1421		− struct vhost_umem umem = kvzalloc(sizeof(umem), GFP_KERNEL);
1422		−
1423		− if (!umem)
1424		− return NULL;
1425		−
1426		− umem->umem_tree = RB_ROOT_CACHED;
1427		− umem->numem = 0;
1428		− INIT_LIST_HEAD(&umem->umem_list);
1429		−
1430		− return umem;
1431		−}
1432		−
1433	1353	static long vhost_set_memory(struct vhost_dev d, struct vhost_memory __user m)
1434	1354	{
1435	1355	struct vhost_memory mem, *newmem;
1436	1356	struct vhost_memory_region *region;
1437		− struct vhost_umem newumem, oldumem;
	1357	+ struct vhost_iotlb newumem, oldumem;
1438	1358	unsigned long size = offsetof(struct vhost_memory, regions);
1439	1359	int i;
1440	1360
@@ -1456,7 +1376,7 @@ static long vhost_set_memory(struct vhost_dev d, struct vhost_memory __user m) · 15 unmodified lines
1456	1376	return -EFAULT;
1457	1377	}
1458	1378
1459		− newumem = vhost_umem_alloc();
	1379	+ newumem = iotlb_alloc();
1460	1380	if (!newumem) {
1461	1381	kvfree(newmem);
1462	1382	return -ENOMEM;
@@ -1465,13 +1385,12 @@ static long vhost_set_memory(struct vhost_dev d, struct vhost_memory __user m) · 2 unmodified lines
1465	1385	for (region = newmem->regions;
1466	1386	region < newmem->regions + mem.nregions;
1467	1387	region++) {
1468		− if (vhost_new_umem_range(newumem,
1469		− region->guest_phys_addr,
1470		− region->memory_size,
1471		− region->guest_phys_addr +
1472		− region->memory_size - 1,
1473		− region->userspace_addr,
1474		− VHOST_ACCESS_RW))
	1388	+ if (vhost_iotlb_add_range(newumem,
	1389	+ region->guest_phys_addr,
	1390	+ region->guest_phys_addr +
	1391	+ region->memory_size - 1,
	1392	+ region->userspace_addr,
	1393	+ VHOST_MAP_RW))
1475	1394	goto err;
1476	1395	}
1477	1396
@@ -1489,11 +1408,11 @@ static long vhost_set_memory(struct vhost_dev d, struct vhost_memory __user m) · 11 unmodified lines
1489	1408	}
1490	1409
1491	1410	kvfree(newmem);
1492		− vhost_umem_clean(oldumem);
	1411	+ vhost_iotlb_free(oldumem);
1493	1412	return 0;
1494	1413
1495	1414	err:
1496		− vhost_umem_clean(newumem);
	1415	+ vhost_iotlb_free(newumem);
1497	1416	kvfree(newmem);
1498	1417	return -EFAULT;
1499	1418	}
@@ -1734,10 +1653,10 @@ EXPORT_SYMBOL_GPL(vhost_vring_ioctl); · 234 unmodified lines
1734	1653
1735	1654	int vhost_init_device_iotlb(struct vhost_dev *d, bool enabled)
1736	1655	{
1737		− struct vhost_umem niotlb, oiotlb;
	1656	+ struct vhost_iotlb niotlb, oiotlb;
1738	1657	int i;
1739	1658
1740		− niotlb = vhost_umem_alloc();
	1659	+ niotlb = iotlb_alloc();
1741	1660	if (!niotlb)
1742	1661	return -ENOMEM;
1743	1662
@@ -1753,7 +1672,7 @@ int vhost_init_device_iotlb(struct vhost_dev *d, bool enabled) · 9 unmodified lines
1753	1672	mutex_unlock(&vq->mutex);
1754	1673	}
1755	1674
1756		− vhost_umem_clean(oiotlb);
	1675	+ vhost_iotlb_free(oiotlb);
1757	1676
1758	1677	return 0;
1759	1678	}
@@ -1883,8 +1802,8 @@ static int log_write(void __user *log_base, · 123 unmodified lines
1883	1802
1884	1803	static int log_write_hva(struct vhost_virtqueue *vq, u64 hva, u64 len)
1885	1804	{
1886		− struct vhost_umem *umem = vq->umem;
1887		− struct vhost_umem_node *u;
	1805	+ struct vhost_iotlb *umem = vq->umem;
	1806	+ struct vhost_iotlb_map *u;
1888	1807	u64 start, end, l, min;
1889	1808	int r;
1890	1809	bool hit = false;
@@ -1894,16 +1813,15 @@ static int log_write_hva(struct vhost_virtqueue *vq, u64 hva, u64 len) · 3 unmodified lines
1894	1813	/* More than one GPAs can be mapped into a single HVA. So
1895	1814	* iterate all possible umems here to be safe.
1896	1815	*/
1897		− list_for_each_entry(u, &umem->umem_list, link) {
1898		− if (u->userspace_addr > hva - 1 + len \|\|
1899		− u->userspace_addr - 1 + u->size < hva)
	1816	+ list_for_each_entry(u, &umem->list, link) {
	1817	+ if (u->addr > hva - 1 + len \|\|
	1818	+ u->addr - 1 + u->size < hva)
1900	1819	continue;
1901		− start = max(u->userspace_addr, hva);
1902		− end = min(u->userspace_addr - 1 + u->size,
1903		− hva - 1 + len);
	1820	+ start = max(u->addr, hva);
	1821	+ end = min(u->addr - 1 + u->size, hva - 1 + len);
1904	1822	l = end - start + 1;
1905	1823	r = log_write(vq->log_base,
1906		− u->start + start - u->userspace_addr,
	1824	+ u->start + start - u->addr,
1907	1825	l);
1908	1826	if (r < 0)
1909	1827	return r;
@@ -2054,9 +1972,9 @@ EXPORT_SYMBOL_GPL(vhost_vq_init_access); · 144 unmodified lines
2054	1972	static int translate_desc(struct vhost_virtqueue *vq, u64 addr, u32 len,
2055	1973	struct iovec iov[], int iov_size, int access)
2056	1974	{
2057		− const struct vhost_umem_node *node;
	1975	+ const struct vhost_iotlb_map *map;
2058	1976	struct vhost_dev *dev = vq->dev;
2059		− struct vhost_umem *umem = dev->iotlb ? dev->iotlb : dev->umem;
	1977	+ struct vhost_iotlb *umem = dev->iotlb ? dev->iotlb : dev->umem;
2060	1978	struct iovec *_iov;
2061	1979	u64 s = 0;
2062	1980	int ret = 0;
@@ -2068,25 +1986,24 @@ static int translate_desc(struct vhost_virtqueue *vq, u64 addr, u32 len, · 5 unmodified lines
2068	1986	break;
2069	1987	}
2070	1988
2071		− node = vhost_umem_interval_tree_iter_first(&umem->umem_tree,
2072		− addr, addr + len - 1);
2073		− if (node == NULL \|\| node->start > addr) {
	1989	+ map = vhost_iotlb_itree_first(umem, addr, addr + len - 1);
	1990	+ if (map == NULL \|\| map->start > addr) {
2074	1991	if (umem != dev->iotlb) {
2075	1992	ret = -EFAULT;
2076	1993	break;
2077	1994	}
2078	1995	ret = -EAGAIN;
2079	1996	break;
2080		− } else if (!(node->perm & access)) {
	1997	+ } else if (!(map->perm & access)) {
2081	1998	ret = -EPERM;
2082	1999	break;
2083	2000	}
2084	2001
2085	2002	_iov = iov + ret;
2086		− size = node->size - addr + node->start;
	2003	+ size = map->size - addr + map->start;
2087	2004	_iov->iov_len = min((u64)len - s, size);
2088	2005	_iov->iov_base = (void __user *)(unsigned long)
2089		− (node->userspace_addr + addr - node->start);
	2006	+ (map->addr + addr - map->start);
2090	2007	s += size;
2091	2008	addr += size;
2092	2009	++ret;

drivers/vhost/vhost.h+11−28

@@ -12,6 +12,7 @@ · 11 unmodified lines
12	12	#include <linux/virtio_config.h>
13	13	#include <linux/virtio_ring.h>
14	14	#include <linux/atomic.h>
	15	+#include <linux/vhost_iotlb.h>
15	16
16	17	struct vhost_work;
17	18	typedef void (vhost_work_fn_t)(struct vhost_work work);
@@ -52,27 +53,6 @@ struct vhost_log { · 34 unmodified lines
52	53	u64 len;
53	54	};
54	55
55		−#define START(node) ((node)->start)
56		−#define LAST(node) ((node)->last)
57		−
58		−struct vhost_umem_node {
59		− struct rb_node rb;
60		− struct list_head link;
61		− __u64 start;
62		− __u64 last;
63		− __u64 size;
64		− __u64 userspace_addr;
65		− __u32 perm;
66		− __u32 flags_padding;
67		− __u64 __subtree_last;
68		−};
69		−
70		−struct vhost_umem {
71		− struct rb_root_cached umem_tree;
72		− struct list_head umem_list;
73		− int numem;
74		−};
75		−
76	56	enum vhost_uaddr_type {
77	57	VHOST_ADDR_DESC = 0,
78	58	VHOST_ADDR_AVAIL = 1,
@@ -90,7 +70,7 @@ struct vhost_virtqueue { · 11 unmodified lines
90	70	struct vring_desc __user *desc;
91	71	struct vring_avail __user *avail;
92	72	struct vring_used __user *used;
93		− const struct vhost_umem_node *meta_iotlb[VHOST_NUM_ADDRS];
	73	+ const struct vhost_iotlb_map *meta_iotlb[VHOST_NUM_ADDRS];
94	74	struct file *kick;
95	75	struct eventfd_ctx *call_ctx;
96	76	struct eventfd_ctx *error_ctx;
@@ -128,8 +108,8 @@ struct vhost_virtqueue { · 31 unmodified lines
128	108	struct iovec *indirect;
129	109	struct vring_used_elem *heads;
130	110	/* Protected by virtqueue mutex. */
131		− struct vhost_umem *umem;
132		− struct vhost_umem *iotlb;
	111	+ struct vhost_iotlb *umem;
	112	+ struct vhost_iotlb *iotlb;
133	113	void *private_data;
134	114	u64 acked_features;
135	115	u64 acked_backend_features;
@@ -164,8 +144,8 @@ struct vhost_dev { · 28 unmodified lines
164	144	struct eventfd_ctx *log_ctx;
165	145	struct llist_head work_list;
166	146	struct task_struct *worker;
167		− struct vhost_umem *umem;
168		− struct vhost_umem *iotlb;
	147	+ struct vhost_iotlb *umem;
	148	+ struct vhost_iotlb *iotlb;
169	149	spinlock_t iotlb_lock;
170	150	struct list_head read_list;
171	151	struct list_head pending_list;
@@ -186,8 +166,8 @@ void vhost_dev_init(struct vhost_dev , struct vhost_virtqueue *vqs, · 14 unmodified lines
186	166	long vhost_dev_set_owner(struct vhost_dev *dev);
187	167	bool vhost_dev_has_owner(struct vhost_dev *dev);
188	168	long vhost_dev_check_owner(struct vhost_dev *);
189		−struct vhost_umem *vhost_dev_reset_owner_prepare(void);
190		−void vhost_dev_reset_owner(struct vhost_dev , struct vhost_umem );
	169	+struct vhost_iotlb *vhost_dev_reset_owner_prepare(void);
	170	+void vhost_dev_reset_owner(struct vhost_dev dev, struct vhost_iotlb iotlb);
191	171	void vhost_dev_cleanup(struct vhost_dev *);
192	172	void vhost_dev_stop(struct vhost_dev *);
193	173	long vhost_dev_ioctl(struct vhost_dev , unsigned int ioctl, void __user argp);
@@ -233,6 +213,9 @@ ssize_t vhost_chr_write_iter(struct vhost_dev *dev, · 39 unmodified lines
233	213	struct iov_iter *from);
234	214	int vhost_init_device_iotlb(struct vhost_dev *d, bool enabled);
235	215
	216	+void vhost_iotlb_map_free(struct vhost_iotlb *iotlb,
	217	+ struct vhost_iotlb_map *map);
	218	+
236	219	#define vq_err(vq, fmt, ...) do { \
237	220	pr_debug(pr_fmt(fmt), ##__VA_ARGS__); \
238	221	if ((vq)->error_ctx) \

include/linux/vhost_iotlb.h+47−0

@@ -0,0 +1,47 @@
	1	+/* SPDX-License-Identifier: GPL-2.0 */
	2	+#ifndef _LINUX_VHOST_IOTLB_H
	3	+#define _LINUX_VHOST_IOTLB_H
	4	+
	5	+#include <linux/interval_tree_generic.h>
	6	+
	7	+struct vhost_iotlb_map {
	8	+ struct rb_node rb;
	9	+ struct list_head link;
	10	+ u64 start;
	11	+ u64 last;
	12	+ u64 size;
	13	+ u64 addr;
	14	+#define VHOST_MAP_RO 0x1
	15	+#define VHOST_MAP_WO 0x2
	16	+#define VHOST_MAP_RW 0x3
	17	+ u32 perm;
	18	+ u32 flags_padding;
	19	+ u64 __subtree_last;
	20	+};
	21	+
	22	+#define VHOST_IOTLB_FLAG_RETIRE 0x1
	23	+
	24	+struct vhost_iotlb {
	25	+ struct rb_root_cached root;
	26	+ struct list_head list;
	27	+ unsigned int limit;
	28	+ unsigned int nmaps;
	29	+ unsigned int flags;
	30	+};
	31	+
	32	+int vhost_iotlb_add_range(struct vhost_iotlb *iotlb, u64 start, u64 last,
	33	+ u64 addr, unsigned int perm);
	34	+void vhost_iotlb_del_range(struct vhost_iotlb *iotlb, u64 start, u64 last);
	35	+
	36	+struct vhost_iotlb *vhost_iotlb_alloc(unsigned int limit, unsigned int flags);
	37	+void vhost_iotlb_free(struct vhost_iotlb *iotlb);
	38	+void vhost_iotlb_reset(struct vhost_iotlb *iotlb);
	39	+
	40	+struct vhost_iotlb_map *
	41	+vhost_iotlb_itree_first(struct vhost_iotlb *iotlb, u64 start, u64 last);
	42	+struct vhost_iotlb_map *
	43	+vhost_iotlb_itree_next(struct vhost_iotlb_map *map, u64 start, u64 last);
	44	+
	45	+void vhost_iotlb_map_free(struct vhost_iotlb *iotlb,
	46	+ struct vhost_iotlb_map *map);
	47	+#endif