History

vhost: factor out IOTLB

This patch factors out IOTLB into a dedicated module in order to be reused by other modules like vringh. User may choose to enable the automatic retiring by specifying VHOST_IOTLB_FLAG_RETIRE flag to fit for the case of vhost device IOTLB implementation.

Jason Wang committed 6 years ago

Commit 0bbe3061 parent 792a4f28 files · +315 −181Browse files

Unified · Split

8 files
  1. MAINTAINERS +1 −0
  2. drivers/vhost/Kconfig +6 −0
  3. drivers/vhost/Makefile +3 −0
  4. drivers/vhost/iotlb.c +177 −0
  5. drivers/vhost/net.c +1 −1
  6. drivers/vhost/vhost.c +69 −152
  7. drivers/vhost/vhost.h +11 −28
  8. include/linux/vhost_iotlb.h +47 −0
MAINTAINERS+1−0
@@ -17766,6 +17766,7 @@ T: git git://git.kernel.org/pub/scm/linux/kernel/git/mst/vhost.git · 17765 unmodified lines
17766S: Maintained17766S: Maintained
17767F: drivers/vhost/17767F: drivers/vhost/
17768F: include/uapi/linux/vhost.h17768F: include/uapi/linux/vhost.h
17769F: include/linux/vhost_iotlb.h
17769 17770
17770VIRTIO INPUT DRIVER17771VIRTIO INPUT DRIVER
17771M: Gerd Hoffmann <kraxel@redhat.com>17772M: Gerd Hoffmann <kraxel@redhat.com>
drivers/vhost/Kconfig+6−0
@@ -1,4 +1,9 @@
1# SPDX-License-Identifier: GPL-2.0-only1# SPDX-License-Identifier: GPL-2.0-only
2config VHOST_IOTLB
3 tristate
4 help
5 Generic IOTLB implementation for vhost and vringh.
6
2config VHOST_RING7config VHOST_RING
3 tristate8 tristate
4 help9 help
@@ -67,4 +72,5 @@ config VHOST_CROSS_ENDIAN_LEGACY · 62 unmodified lines
67 adds some overhead, it is disabled by default.72 adds some overhead, it is disabled by default.
68 73
69 If unsure, say "N".74 If unsure, say "N".
75
70endif76endif
drivers/vhost/Makefile+3−0
@@ -11,3 +11,6 @@ vhost_vsock-y := vsock.o · 10 unmodified lines
11obj-$(CONFIG_VHOST_RING) += vringh.o11obj-$(CONFIG_VHOST_RING) += vringh.o
12 12
13obj-$(CONFIG_VHOST) += vhost.o13obj-$(CONFIG_VHOST) += vhost.o
14
15obj-$(CONFIG_VHOST_IOTLB) += vhost_iotlb.o
16vhost_iotlb-y := iotlb.o
drivers/vhost/iotlb.c+177−0
@@ -0,0 +1,177 @@
1// SPDX-License-Identifier: GPL-2.0-only
2/* Copyright (C) 2020 Red Hat, Inc.
3 * Author: Jason Wang <jasowang@redhat.com>
4 *
5 * IOTLB implementation for vhost.
6 */
7#include <linux/slab.h>
8#include <linux/vhost_iotlb.h>
9#include <linux/module.h>
10
11#define MOD_VERSION "0.1"
12#define MOD_DESC "VHOST IOTLB"
13#define MOD_AUTHOR "Jason Wang <jasowang@redhat.com>"
14#define MOD_LICENSE "GPL v2"
15
16#define START(map) ((map)->start)
17#define LAST(map) ((map)->last)
18
19INTERVAL_TREE_DEFINE(struct vhost_iotlb_map,
20 rb, __u64, __subtree_last,
21 START, LAST, static inline, vhost_iotlb_itree);
22
23/**
24 * vhost_iotlb_map_free - remove a map node and free it
25 * @iotlb: the IOTLB
26 * @map: the map that want to be remove and freed
27 */
28void vhost_iotlb_map_free(struct vhost_iotlb *iotlb,
29 struct vhost_iotlb_map *map)
30{
31 vhost_iotlb_itree_remove(map, &iotlb->root);
32 list_del(&map->link);
33 kfree(map);
34 iotlb->nmaps--;
35}
36EXPORT_SYMBOL_GPL(vhost_iotlb_map_free);
37
38/**
39 * vhost_iotlb_add_range - add a new range to vhost IOTLB
40 * @iotlb: the IOTLB
41 * @start: start of the IOVA range
42 * @last: last of IOVA range
43 * @addr: the address that is mapped to @start
44 * @perm: access permission of this range
45 *
46 * Returns an error last is smaller than start or memory allocation
47 * fails
48 */
49int vhost_iotlb_add_range(struct vhost_iotlb *iotlb,
50 u64 start, u64 last,
51 u64 addr, unsigned int perm)
52{
53 struct vhost_iotlb_map *map;
54
55 if (last < start)
56 return -EFAULT;
57
58 if (iotlb->limit &&
59 iotlb->nmaps == iotlb->limit &&
60 iotlb->flags & VHOST_IOTLB_FLAG_RETIRE) {
61 map = list_first_entry(&iotlb->list, typeof(*map), link);
62 vhost_iotlb_map_free(iotlb, map);
63 }
64
65 map = kmalloc(sizeof(*map), GFP_ATOMIC);
66 if (!map)
67 return -ENOMEM;
68
69 map->start = start;
70 map->size = last - start + 1;
71 map->last = last;
72 map->addr = addr;
73 map->perm = perm;
74
75 iotlb->nmaps++;
76 vhost_iotlb_itree_insert(map, &iotlb->root);
77
78 INIT_LIST_HEAD(&map->link);
79 list_add_tail(&map->link, &iotlb->list);
80
81 return 0;
82}
83EXPORT_SYMBOL_GPL(vhost_iotlb_add_range);
84
85/**
86 * vring_iotlb_del_range - delete overlapped ranges from vhost IOTLB
87 * @iotlb: the IOTLB
88 * @start: start of the IOVA range
89 * @last: last of IOVA range
90 */
91void vhost_iotlb_del_range(struct vhost_iotlb *iotlb, u64 start, u64 last)
92{
93 struct vhost_iotlb_map *map;
94
95 while ((map = vhost_iotlb_itree_iter_first(&iotlb->root,
96 start, last)))
97 vhost_iotlb_map_free(iotlb, map);
98}
99EXPORT_SYMBOL_GPL(vhost_iotlb_del_range);
100
101/**
102 * vhost_iotlb_alloc - add a new vhost IOTLB
103 * @limit: maximum number of IOTLB entries
104 * @flags: VHOST_IOTLB_FLAG_XXX
105 *
106 * Returns an error is memory allocation fails
107 */
108struct vhost_iotlb *vhost_iotlb_alloc(unsigned int limit, unsigned int flags)
109{
110 struct vhost_iotlb *iotlb = kzalloc(sizeof(*iotlb), GFP_KERNEL);
111
112 if (!iotlb)
113 return NULL;
114
115 iotlb->root = RB_ROOT_CACHED;
116 iotlb->limit = limit;
117 iotlb->nmaps = 0;
118 iotlb->flags = flags;
119 INIT_LIST_HEAD(&iotlb->list);
120
121 return iotlb;
122}
123EXPORT_SYMBOL_GPL(vhost_iotlb_alloc);
124
125/**
126 * vhost_iotlb_reset - reset vhost IOTLB (free all IOTLB entries)
127 * @iotlb: the IOTLB to be reset
128 */
129void vhost_iotlb_reset(struct vhost_iotlb *iotlb)
130{
131 vhost_iotlb_del_range(iotlb, 0ULL, 0ULL - 1);
132}
133EXPORT_SYMBOL_GPL(vhost_iotlb_reset);
134
135/**
136 * vhost_iotlb_free - reset and free vhost IOTLB
137 * @iotlb: the IOTLB to be freed
138 */
139void vhost_iotlb_free(struct vhost_iotlb *iotlb)
140{
141 if (iotlb) {
142 vhost_iotlb_reset(iotlb);
143 kfree(iotlb);
144 }
145}
146EXPORT_SYMBOL_GPL(vhost_iotlb_free);
147
148/**
149 * vhost_iotlb_itree_first - return the first overlapped range
150 * @iotlb: the IOTLB
151 * @start: start of IOVA range
152 * @end: end of IOVA range
153 */
154struct vhost_iotlb_map *
155vhost_iotlb_itree_first(struct vhost_iotlb *iotlb, u64 start, u64 last)
156{
157 return vhost_iotlb_itree_iter_first(&iotlb->root, start, last);
158}
159EXPORT_SYMBOL_GPL(vhost_iotlb_itree_first);
160
161/**
162 * vhost_iotlb_itree_first - return the next overlapped range
163 * @iotlb: the IOTLB
164 * @start: start of IOVA range
165 * @end: end of IOVA range
166 */
167struct vhost_iotlb_map *
168vhost_iotlb_itree_next(struct vhost_iotlb_map *map, u64 start, u64 last)
169{
170 return vhost_iotlb_itree_iter_next(map, start, last);
171}
172EXPORT_SYMBOL_GPL(vhost_iotlb_itree_next);
173
174MODULE_VERSION(MOD_VERSION);
175MODULE_DESCRIPTION(MOD_DESC);
176MODULE_AUTHOR(MOD_AUTHOR);
177MODULE_LICENSE(MOD_LICENSE);
drivers/vhost/net.c+1−1
@@ -1587,7 +1587,7 @@ static long vhost_net_reset_owner(struct vhost_net *n) · 1586 unmodified lines
1587 struct socket *tx_sock = NULL;1587 struct socket *tx_sock = NULL;
1588 struct socket *rx_sock = NULL;1588 struct socket *rx_sock = NULL;
1589 long err;1589 long err;
1590 struct vhost_umem *umem;1590 struct vhost_iotlb *umem;
1591 1591
1592 mutex_lock(&n->dev.mutex);1592 mutex_lock(&n->dev.mutex);
1593 err = vhost_dev_check_owner(&n->dev);1593 err = vhost_dev_check_owner(&n->dev);
drivers/vhost/vhost.c+69−152
@@ -50,10 +50,6 @@ enum { · 49 unmodified lines
50#define vhost_used_event(vq) ((__virtio16 __user *)&vq->avail->ring[vq->num])50#define vhost_used_event(vq) ((__virtio16 __user *)&vq->avail->ring[vq->num])
51#define vhost_avail_event(vq) ((__virtio16 __user *)&vq->used->ring[vq->num])51#define vhost_avail_event(vq) ((__virtio16 __user *)&vq->used->ring[vq->num])
52 52
53INTERVAL_TREE_DEFINE(struct vhost_umem_node,
54 rb, __u64, __subtree_last,
55 START, LAST, static inline, vhost_umem_interval_tree);
56
57#ifdef CONFIG_VHOST_CROSS_ENDIAN_LEGACY53#ifdef CONFIG_VHOST_CROSS_ENDIAN_LEGACY
58static void vhost_disable_cross_endian(struct vhost_virtqueue *vq)54static void vhost_disable_cross_endian(struct vhost_virtqueue *vq)
59{55{
@@ -584,21 +580,25 @@ err_mm: · 524 unmodified lines
584}580}
585EXPORT_SYMBOL_GPL(vhost_dev_set_owner);581EXPORT_SYMBOL_GPL(vhost_dev_set_owner);
586 582
587struct vhost_umem *vhost_dev_reset_owner_prepare(void)583static struct vhost_iotlb *iotlb_alloc(void)
584{
585 return vhost_iotlb_alloc(max_iotlb_entries,
586 VHOST_IOTLB_FLAG_RETIRE);
587}
588
589struct vhost_iotlb *vhost_dev_reset_owner_prepare(void)
588{590{
589 return kvzalloc(sizeof(struct vhost_umem), GFP_KERNEL);591 return iotlb_alloc();
590}592}
591EXPORT_SYMBOL_GPL(vhost_dev_reset_owner_prepare);593EXPORT_SYMBOL_GPL(vhost_dev_reset_owner_prepare);
592 594
593/* Caller should have device mutex */595/* Caller should have device mutex */
594void vhost_dev_reset_owner(struct vhost_dev *dev, struct vhost_umem *umem)596void vhost_dev_reset_owner(struct vhost_dev *dev, struct vhost_iotlb *umem)
595{597{
596 int i;598 int i;
597 599
598 vhost_dev_cleanup(dev);600 vhost_dev_cleanup(dev);
599 601
600 /* Restore memory to default empty mapping. */
601 INIT_LIST_HEAD(&umem->umem_list);
602 dev->umem = umem;602 dev->umem = umem;
603 /* We don't need VQ locks below since vhost_dev_cleanup makes sure603 /* We don't need VQ locks below since vhost_dev_cleanup makes sure
604 * VQs aren't running.604 * VQs aren't running.
@@ -621,28 +621,6 @@ void vhost_dev_stop(struct vhost_dev *dev) · 16 unmodified lines
621}621}
622EXPORT_SYMBOL_GPL(vhost_dev_stop);622EXPORT_SYMBOL_GPL(vhost_dev_stop);
623 623
624static void vhost_umem_free(struct vhost_umem *umem,
625 struct vhost_umem_node *node)
626{
627 vhost_umem_interval_tree_remove(node, &umem->umem_tree);
628 list_del(&node->link);
629 kfree(node);
630 umem->numem--;
631}
632
633static void vhost_umem_clean(struct vhost_umem *umem)
634{
635 struct vhost_umem_node *node, *tmp;
636
637 if (!umem)
638 return;
639
640 list_for_each_entry_safe(node, tmp, &umem->umem_list, link)
641 vhost_umem_free(umem, node);
642
643 kvfree(umem);
644}
645
646static void vhost_clear_msg(struct vhost_dev *dev)624static void vhost_clear_msg(struct vhost_dev *dev)
647{625{
648 struct vhost_msg_node *node, *n;626 struct vhost_msg_node *node, *n;
@@ -680,9 +658,9 @@ void vhost_dev_cleanup(struct vhost_dev *dev) · 31 unmodified lines
680 eventfd_ctx_put(dev->log_ctx);658 eventfd_ctx_put(dev->log_ctx);
681 dev->log_ctx = NULL;659 dev->log_ctx = NULL;
682 /* No one will access memory at this point */660 /* No one will access memory at this point */
683 vhost_umem_clean(dev->umem);661 vhost_iotlb_free(dev->umem);
684 dev->umem = NULL;662 dev->umem = NULL;
685 vhost_umem_clean(dev->iotlb);663 vhost_iotlb_free(dev->iotlb);
686 dev->iotlb = NULL;664 dev->iotlb = NULL;
687 vhost_clear_msg(dev);665 vhost_clear_msg(dev);
688 wake_up_interruptible_poll(&dev->wait, EPOLLIN | EPOLLRDNORM);666 wake_up_interruptible_poll(&dev->wait, EPOLLIN | EPOLLRDNORM);
@@ -718,27 +696,26 @@ static bool vhost_overflow(u64 uaddr, u64 size) · 29 unmodified lines
718}696}
719 697
720/* Caller should have vq mutex and device mutex. */698/* Caller should have vq mutex and device mutex. */
721static bool vq_memory_access_ok(void __user *log_base, struct vhost_umem *umem,699static bool vq_memory_access_ok(void __user *log_base, struct vhost_iotlb *umem,
722 int log_all)700 int log_all)
723{701{
724 struct vhost_umem_node *node;702 struct vhost_iotlb_map *map;
725 703
726 if (!umem)704 if (!umem)
727 return false;705 return false;
728 706
729 list_for_each_entry(node, &umem->umem_list, link) {707 list_for_each_entry(map, &umem->list, link) {
730 unsigned long a = node->userspace_addr;708 unsigned long a = map->addr;
731 709
732 if (vhost_overflow(node->userspace_addr, node->size))710 if (vhost_overflow(map->addr, map->size))
733 return false;711 return false;
734 712
735 713
736 if (!access_ok((void __user *)a,714 if (!access_ok((void __user *)a, map->size))
737 node->size))
738 return false;715 return false;
739 else if (log_all && !log_access_ok(log_base,716 else if (log_all && !log_access_ok(log_base,
740 node->start,717 map->start,
741 node->size))718 map->size))
742 return false;719 return false;
743 }720 }
744 return true;721 return true;
@@ -748,17 +725,17 @@ static inline void __user *vhost_vq_meta_fetch(struct vhost_virtqueue *vq, · 3 unmodified lines
748 u64 addr, unsigned int size,725 u64 addr, unsigned int size,
749 int type)726 int type)
750{727{
751 const struct vhost_umem_node *node = vq->meta_iotlb[type];728 const struct vhost_iotlb_map *map = vq->meta_iotlb[type];
752 729
753 if (!node)730 if (!map)
754 return NULL;731 return NULL;
755 732
756 return (void *)(uintptr_t)(node->userspace_addr + addr - node->start);733 return (void *)(uintptr_t)(map->addr + addr - map->start);
757}734}
758 735
759/* Can we switch to this memory table? */736/* Can we switch to this memory table? */
760/* Caller should have device mutex but not vq mutex */737/* Caller should have device mutex but not vq mutex */
761static bool memory_access_ok(struct vhost_dev *d, struct vhost_umem *umem,738static bool memory_access_ok(struct vhost_dev *d, struct vhost_iotlb *umem,
762 int log_all)739 int log_all)
763{740{
764 int i;741 int i;
@@ -1023,47 +1000,6 @@ static inline int vhost_get_desc(struct vhost_virtqueue *vq, · 258 unmodified lines
1023 return vhost_copy_from_user(vq, desc, vq->desc + idx, sizeof(*desc));1000 return vhost_copy_from_user(vq, desc, vq->desc + idx, sizeof(*desc));
1024}1001}
1025 1002
1026static int vhost_new_umem_range(struct vhost_umem *umem,
1027 u64 start, u64 size, u64 end,
1028 u64 userspace_addr, int perm)
1029{
1030 struct vhost_umem_node *tmp, *node;
1031
1032 if (!size)
1033 return -EFAULT;
1034
1035 node = kmalloc(sizeof(*node), GFP_ATOMIC);
1036 if (!node)
1037 return -ENOMEM;
1038
1039 if (umem->numem == max_iotlb_entries) {
1040 tmp = list_first_entry(&umem->umem_list, typeof(*tmp), link);
1041 vhost_umem_free(umem, tmp);
1042 }
1043
1044 node->start = start;
1045 node->size = size;
1046 node->last = end;
1047 node->userspace_addr = userspace_addr;
1048 node->perm = perm;
1049 INIT_LIST_HEAD(&node->link);
1050 list_add_tail(&node->link, &umem->umem_list);
1051 vhost_umem_interval_tree_insert(node, &umem->umem_tree);
1052 umem->numem++;
1053
1054 return 0;
1055}
1056
1057static void vhost_del_umem_range(struct vhost_umem *umem,
1058 u64 start, u64 end)
1059{
1060 struct vhost_umem_node *node;
1061
1062 while ((node = vhost_umem_interval_tree_iter_first(&umem->umem_tree,
1063 start, end)))
1064 vhost_umem_free(umem, node);
1065}
1066
1067static void vhost_iotlb_notify_vq(struct vhost_dev *d,1003static void vhost_iotlb_notify_vq(struct vhost_dev *d,
1068 struct vhost_iotlb_msg *msg)1004 struct vhost_iotlb_msg *msg)
1069{1005{
@@ -1120,9 +1056,9 @@ static int vhost_process_iotlb_msg(struct vhost_dev *dev, · 50 unmodified lines
1120 break;1056 break;
1121 }1057 }
1122 vhost_vq_meta_reset(dev);1058 vhost_vq_meta_reset(dev);
1123 if (vhost_new_umem_range(dev->iotlb, msg->iova, msg->size,1059 if (vhost_iotlb_add_range(dev->iotlb, msg->iova,
1124 msg->iova + msg->size - 1,1060 msg->iova + msg->size - 1,
1125 msg->uaddr, msg->perm)) {1061 msg->uaddr, msg->perm)) {
1126 ret = -ENOMEM;1062 ret = -ENOMEM;
1127 break;1063 break;
1128 }1064 }
@@ -1134,8 +1070,8 @@ static int vhost_process_iotlb_msg(struct vhost_dev *dev, · 5 unmodified lines
1134 break;1070 break;
1135 }1071 }
1136 vhost_vq_meta_reset(dev);1072 vhost_vq_meta_reset(dev);
1137 vhost_del_umem_range(dev->iotlb, msg->iova,1073 vhost_iotlb_del_range(dev->iotlb, msg->iova,
1138 msg->iova + msg->size - 1);1074 msg->iova + msg->size - 1);
1139 break;1075 break;
1140 default:1076 default:
1141 ret = -EINVAL;1077 ret = -EINVAL;
@@ -1319,44 +1255,42 @@ static bool vq_access_ok(struct vhost_virtqueue *vq, unsigned int num, · 177 unmodified lines
1319}1255}
1320 1256
1321static void vhost_vq_meta_update(struct vhost_virtqueue *vq,1257static void vhost_vq_meta_update(struct vhost_virtqueue *vq,
1322 const struct vhost_umem_node *node,1258 const struct vhost_iotlb_map *map,
1323 int type)1259 int type)
1324{1260{
1325 int access = (type == VHOST_ADDR_USED) ?1261 int access = (type == VHOST_ADDR_USED) ?
1326 VHOST_ACCESS_WO : VHOST_ACCESS_RO;1262 VHOST_ACCESS_WO : VHOST_ACCESS_RO;
1327 1263
1328 if (likely(node->perm & access))1264 if (likely(map->perm & access))
1329 vq->meta_iotlb[type] = node;1265 vq->meta_iotlb[type] = map;
1330}1266}
1331 1267
1332static bool iotlb_access_ok(struct vhost_virtqueue *vq,1268static bool iotlb_access_ok(struct vhost_virtqueue *vq,
1333 int access, u64 addr, u64 len, int type)1269 int access, u64 addr, u64 len, int type)
1334{1270{
1335 const struct vhost_umem_node *node;1271 const struct vhost_iotlb_map *map;
1336 struct vhost_umem *umem = vq->iotlb;1272 struct vhost_iotlb *umem = vq->iotlb;
1337 u64 s = 0, size, orig_addr = addr, last = addr + len - 1;1273 u64 s = 0, size, orig_addr = addr, last = addr + len - 1;
1338 1274
1339 if (vhost_vq_meta_fetch(vq, addr, len, type))1275 if (vhost_vq_meta_fetch(vq, addr, len, type))
1340 return true;1276 return true;
1341 1277
1342 while (len > s) {1278 while (len > s) {
1343 node = vhost_umem_interval_tree_iter_first(&umem->umem_tree,1279 map = vhost_iotlb_itree_first(umem, addr, last);
1344 addr,1280 if (map == NULL || map->start > addr) {
1345 last);
1346 if (node == NULL || node->start > addr) {
1347 vhost_iotlb_miss(vq, addr, access);1281 vhost_iotlb_miss(vq, addr, access);
1348 return false;1282 return false;
1349 } else if (!(node->perm & access)) {1283 } else if (!(map->perm & access)) {
1350 /* Report the possible access violation by1284 /* Report the possible access violation by
1351 * request another translation from userspace.1285 * request another translation from userspace.
1352 */1286 */
1353 return false;1287 return false;
1354 }1288 }
1355 1289
1356 size = node->size - addr + node->start;1290 size = map->size - addr + map->start;
1357 1291
1358 if (orig_addr == addr && size >= len)1292 if (orig_addr == addr && size >= len)
1359 vhost_vq_meta_update(vq, node, type);1293 vhost_vq_meta_update(vq, map, type);
1360 1294
1361 s += size;1295 s += size;
1362 addr += size;1296 addr += size;
@@ -1372,12 +1306,12 @@ int vq_meta_prefetch(struct vhost_virtqueue *vq) · 9 unmodified lines
1372 if (!vq->iotlb)1306 if (!vq->iotlb)
1373 return 1;1307 return 1;
1374 1308
1375 return iotlb_access_ok(vq, VHOST_ACCESS_RO, (u64)(uintptr_t)vq->desc,1309 return iotlb_access_ok(vq, VHOST_MAP_RO, (u64)(uintptr_t)vq->desc,
1376 vhost_get_desc_size(vq, num), VHOST_ADDR_DESC) &&1310 vhost_get_desc_size(vq, num), VHOST_ADDR_DESC) &&
1377 iotlb_access_ok(vq, VHOST_ACCESS_RO, (u64)(uintptr_t)vq->avail,1311 iotlb_access_ok(vq, VHOST_MAP_RO, (u64)(uintptr_t)vq->avail,
1378 vhost_get_avail_size(vq, num),1312 vhost_get_avail_size(vq, num),
1379 VHOST_ADDR_AVAIL) &&1313 VHOST_ADDR_AVAIL) &&
1380 iotlb_access_ok(vq, VHOST_ACCESS_WO, (u64)(uintptr_t)vq->used,1314 iotlb_access_ok(vq, VHOST_MAP_WO, (u64)(uintptr_t)vq->used,
1381 vhost_get_used_size(vq, num), VHOST_ADDR_USED);1315 vhost_get_used_size(vq, num), VHOST_ADDR_USED);
1382}1316}
1383EXPORT_SYMBOL_GPL(vq_meta_prefetch);1317EXPORT_SYMBOL_GPL(vq_meta_prefetch);
@@ -1416,25 +1350,11 @@ bool vhost_vq_access_ok(struct vhost_virtqueue *vq) · 32 unmodified lines
1416}1350}
1417EXPORT_SYMBOL_GPL(vhost_vq_access_ok);1351EXPORT_SYMBOL_GPL(vhost_vq_access_ok);
1418 1352
1419static struct vhost_umem *vhost_umem_alloc(void)
1420{
1421 struct vhost_umem *umem = kvzalloc(sizeof(*umem), GFP_KERNEL);
1422
1423 if (!umem)
1424 return NULL;
1425
1426 umem->umem_tree = RB_ROOT_CACHED;
1427 umem->numem = 0;
1428 INIT_LIST_HEAD(&umem->umem_list);
1429
1430 return umem;
1431}
1432
1433static long vhost_set_memory(struct vhost_dev *d, struct vhost_memory __user *m)1353static long vhost_set_memory(struct vhost_dev *d, struct vhost_memory __user *m)
1434{1354{
1435 struct vhost_memory mem, *newmem;1355 struct vhost_memory mem, *newmem;
1436 struct vhost_memory_region *region;1356 struct vhost_memory_region *region;
1437 struct vhost_umem *newumem, *oldumem;1357 struct vhost_iotlb *newumem, *oldumem;
1438 unsigned long size = offsetof(struct vhost_memory, regions);1358 unsigned long size = offsetof(struct vhost_memory, regions);
1439 int i;1359 int i;
1440 1360
@@ -1456,7 +1376,7 @@ static long vhost_set_memory(struct vhost_dev *d, struct vhost_memory __user *m) · 15 unmodified lines
1456 return -EFAULT;1376 return -EFAULT;
1457 }1377 }
1458 1378
1459 newumem = vhost_umem_alloc();1379 newumem = iotlb_alloc();
1460 if (!newumem) {1380 if (!newumem) {
1461 kvfree(newmem);1381 kvfree(newmem);
1462 return -ENOMEM;1382 return -ENOMEM;
@@ -1465,13 +1385,12 @@ static long vhost_set_memory(struct vhost_dev *d, struct vhost_memory __user *m) · 2 unmodified lines
1465 for (region = newmem->regions;1385 for (region = newmem->regions;
1466 region < newmem->regions + mem.nregions;1386 region < newmem->regions + mem.nregions;
1467 region++) {1387 region++) {
1468 if (vhost_new_umem_range(newumem,1388 if (vhost_iotlb_add_range(newumem,
1469 region->guest_phys_addr,1389 region->guest_phys_addr,
1470 region->memory_size,1390 region->guest_phys_addr +
1471 region->guest_phys_addr +1391 region->memory_size - 1,
1472 region->memory_size - 1,1392 region->userspace_addr,
1473 region->userspace_addr,1393 VHOST_MAP_RW))
1474 VHOST_ACCESS_RW))
1475 goto err;1394 goto err;
1476 }1395 }
1477 1396
@@ -1489,11 +1408,11 @@ static long vhost_set_memory(struct vhost_dev *d, struct vhost_memory __user *m) · 11 unmodified lines
1489 }1408 }
1490 1409
1491 kvfree(newmem);1410 kvfree(newmem);
1492 vhost_umem_clean(oldumem);1411 vhost_iotlb_free(oldumem);
1493 return 0;1412 return 0;
1494 1413
1495err:1414err:
1496 vhost_umem_clean(newumem);1415 vhost_iotlb_free(newumem);
1497 kvfree(newmem);1416 kvfree(newmem);
1498 return -EFAULT;1417 return -EFAULT;
1499}1418}
@@ -1734,10 +1653,10 @@ EXPORT_SYMBOL_GPL(vhost_vring_ioctl); · 234 unmodified lines
1734 1653
1735int vhost_init_device_iotlb(struct vhost_dev *d, bool enabled)1654int vhost_init_device_iotlb(struct vhost_dev *d, bool enabled)
1736{1655{
1737 struct vhost_umem *niotlb, *oiotlb;1656 struct vhost_iotlb *niotlb, *oiotlb;
1738 int i;1657 int i;
1739 1658
1740 niotlb = vhost_umem_alloc();1659 niotlb = iotlb_alloc();
1741 if (!niotlb)1660 if (!niotlb)
1742 return -ENOMEM;1661 return -ENOMEM;
1743 1662
@@ -1753,7 +1672,7 @@ int vhost_init_device_iotlb(struct vhost_dev *d, bool enabled) · 9 unmodified lines
1753 mutex_unlock(&vq->mutex);1672 mutex_unlock(&vq->mutex);
1754 }1673 }
1755 1674
1756 vhost_umem_clean(oiotlb);1675 vhost_iotlb_free(oiotlb);
1757 1676
1758 return 0;1677 return 0;
1759}1678}
@@ -1883,8 +1802,8 @@ static int log_write(void __user *log_base, · 123 unmodified lines
1883 1802
1884static int log_write_hva(struct vhost_virtqueue *vq, u64 hva, u64 len)1803static int log_write_hva(struct vhost_virtqueue *vq, u64 hva, u64 len)
1885{1804{
1886 struct vhost_umem *umem = vq->umem;1805 struct vhost_iotlb *umem = vq->umem;
1887 struct vhost_umem_node *u;1806 struct vhost_iotlb_map *u;
1888 u64 start, end, l, min;1807 u64 start, end, l, min;
1889 int r;1808 int r;
1890 bool hit = false;1809 bool hit = false;
@@ -1894,16 +1813,15 @@ static int log_write_hva(struct vhost_virtqueue *vq, u64 hva, u64 len) · 3 unmodified lines
1894 /* More than one GPAs can be mapped into a single HVA. So1813 /* More than one GPAs can be mapped into a single HVA. So
1895 * iterate all possible umems here to be safe.1814 * iterate all possible umems here to be safe.
1896 */1815 */
1897 list_for_each_entry(u, &umem->umem_list, link) {1816 list_for_each_entry(u, &umem->list, link) {
1898 if (u->userspace_addr > hva - 1 + len ||1817 if (u->addr > hva - 1 + len ||
1899 u->userspace_addr - 1 + u->size < hva)1818 u->addr - 1 + u->size < hva)
1900 continue;1819 continue;
1901 start = max(u->userspace_addr, hva);1820 start = max(u->addr, hva);
1902 end = min(u->userspace_addr - 1 + u->size,1821 end = min(u->addr - 1 + u->size, hva - 1 + len);
1903 hva - 1 + len);
1904 l = end - start + 1;1822 l = end - start + 1;
1905 r = log_write(vq->log_base,1823 r = log_write(vq->log_base,
1906 u->start + start - u->userspace_addr,1824 u->start + start - u->addr,
1907 l);1825 l);
1908 if (r < 0)1826 if (r < 0)
1909 return r;1827 return r;
@@ -2054,9 +1972,9 @@ EXPORT_SYMBOL_GPL(vhost_vq_init_access); · 144 unmodified lines
2054static int translate_desc(struct vhost_virtqueue *vq, u64 addr, u32 len,1972static int translate_desc(struct vhost_virtqueue *vq, u64 addr, u32 len,
2055 struct iovec iov[], int iov_size, int access)1973 struct iovec iov[], int iov_size, int access)
2056{1974{
2057 const struct vhost_umem_node *node;1975 const struct vhost_iotlb_map *map;
2058 struct vhost_dev *dev = vq->dev;1976 struct vhost_dev *dev = vq->dev;
2059 struct vhost_umem *umem = dev->iotlb ? dev->iotlb : dev->umem;1977 struct vhost_iotlb *umem = dev->iotlb ? dev->iotlb : dev->umem;
2060 struct iovec *_iov;1978 struct iovec *_iov;
2061 u64 s = 0;1979 u64 s = 0;
2062 int ret = 0;1980 int ret = 0;
@@ -2068,25 +1986,24 @@ static int translate_desc(struct vhost_virtqueue *vq, u64 addr, u32 len, · 5 unmodified lines
2068 break;1986 break;
2069 }1987 }
2070 1988
2071 node = vhost_umem_interval_tree_iter_first(&umem->umem_tree,1989 map = vhost_iotlb_itree_first(umem, addr, addr + len - 1);
2072 addr, addr + len - 1);1990 if (map == NULL || map->start > addr) {
2073 if (node == NULL || node->start > addr) {
2074 if (umem != dev->iotlb) {1991 if (umem != dev->iotlb) {
2075 ret = -EFAULT;1992 ret = -EFAULT;
2076 break;1993 break;
2077 }1994 }
2078 ret = -EAGAIN;1995 ret = -EAGAIN;
2079 break;1996 break;
2080 } else if (!(node->perm & access)) {1997 } else if (!(map->perm & access)) {
2081 ret = -EPERM;1998 ret = -EPERM;
2082 break;1999 break;
2083 }2000 }
2084 2001
2085 _iov = iov + ret;2002 _iov = iov + ret;
2086 size = node->size - addr + node->start;2003 size = map->size - addr + map->start;
2087 _iov->iov_len = min((u64)len - s, size);2004 _iov->iov_len = min((u64)len - s, size);
2088 _iov->iov_base = (void __user *)(unsigned long)2005 _iov->iov_base = (void __user *)(unsigned long)
2089 (node->userspace_addr + addr - node->start);2006 (map->addr + addr - map->start);
2090 s += size;2007 s += size;
2091 addr += size;2008 addr += size;
2092 ++ret;2009 ++ret;
drivers/vhost/vhost.h+11−28
@@ -12,6 +12,7 @@ · 11 unmodified lines
12#include <linux/virtio_config.h>12#include <linux/virtio_config.h>
13#include <linux/virtio_ring.h>13#include <linux/virtio_ring.h>
14#include <linux/atomic.h>14#include <linux/atomic.h>
15#include <linux/vhost_iotlb.h>
15 16
16struct vhost_work;17struct vhost_work;
17typedef void (*vhost_work_fn_t)(struct vhost_work *work);18typedef void (*vhost_work_fn_t)(struct vhost_work *work);
@@ -52,27 +53,6 @@ struct vhost_log { · 34 unmodified lines
52 u64 len;53 u64 len;
53};54};
54 55
55#define START(node) ((node)->start)
56#define LAST(node) ((node)->last)
57
58struct vhost_umem_node {
59 struct rb_node rb;
60 struct list_head link;
61 __u64 start;
62 __u64 last;
63 __u64 size;
64 __u64 userspace_addr;
65 __u32 perm;
66 __u32 flags_padding;
67 __u64 __subtree_last;
68};
69
70struct vhost_umem {
71 struct rb_root_cached umem_tree;
72 struct list_head umem_list;
73 int numem;
74};
75
76enum vhost_uaddr_type {56enum vhost_uaddr_type {
77 VHOST_ADDR_DESC = 0,57 VHOST_ADDR_DESC = 0,
78 VHOST_ADDR_AVAIL = 1,58 VHOST_ADDR_AVAIL = 1,
@@ -90,7 +70,7 @@ struct vhost_virtqueue { · 11 unmodified lines
90 struct vring_desc __user *desc;70 struct vring_desc __user *desc;
91 struct vring_avail __user *avail;71 struct vring_avail __user *avail;
92 struct vring_used __user *used;72 struct vring_used __user *used;
93 const struct vhost_umem_node *meta_iotlb[VHOST_NUM_ADDRS];73 const struct vhost_iotlb_map *meta_iotlb[VHOST_NUM_ADDRS];
94 struct file *kick;74 struct file *kick;
95 struct eventfd_ctx *call_ctx;75 struct eventfd_ctx *call_ctx;
96 struct eventfd_ctx *error_ctx;76 struct eventfd_ctx *error_ctx;
@@ -128,8 +108,8 @@ struct vhost_virtqueue { · 31 unmodified lines
128 struct iovec *indirect;108 struct iovec *indirect;
129 struct vring_used_elem *heads;109 struct vring_used_elem *heads;
130 /* Protected by virtqueue mutex. */110 /* Protected by virtqueue mutex. */
131 struct vhost_umem *umem;111 struct vhost_iotlb *umem;
132 struct vhost_umem *iotlb;112 struct vhost_iotlb *iotlb;
133 void *private_data;113 void *private_data;
134 u64 acked_features;114 u64 acked_features;
135 u64 acked_backend_features;115 u64 acked_backend_features;
@@ -164,8 +144,8 @@ struct vhost_dev { · 28 unmodified lines
164 struct eventfd_ctx *log_ctx;144 struct eventfd_ctx *log_ctx;
165 struct llist_head work_list;145 struct llist_head work_list;
166 struct task_struct *worker;146 struct task_struct *worker;
167 struct vhost_umem *umem;147 struct vhost_iotlb *umem;
168 struct vhost_umem *iotlb;148 struct vhost_iotlb *iotlb;
169 spinlock_t iotlb_lock;149 spinlock_t iotlb_lock;
170 struct list_head read_list;150 struct list_head read_list;
171 struct list_head pending_list;151 struct list_head pending_list;
@@ -186,8 +166,8 @@ void vhost_dev_init(struct vhost_dev *, struct vhost_virtqueue **vqs, · 14 unmodified lines
186long vhost_dev_set_owner(struct vhost_dev *dev);166long vhost_dev_set_owner(struct vhost_dev *dev);
187bool vhost_dev_has_owner(struct vhost_dev *dev);167bool vhost_dev_has_owner(struct vhost_dev *dev);
188long vhost_dev_check_owner(struct vhost_dev *);168long vhost_dev_check_owner(struct vhost_dev *);
189struct vhost_umem *vhost_dev_reset_owner_prepare(void);169struct vhost_iotlb *vhost_dev_reset_owner_prepare(void);
190void vhost_dev_reset_owner(struct vhost_dev *, struct vhost_umem *);170void vhost_dev_reset_owner(struct vhost_dev *dev, struct vhost_iotlb *iotlb);
191void vhost_dev_cleanup(struct vhost_dev *);171void vhost_dev_cleanup(struct vhost_dev *);
192void vhost_dev_stop(struct vhost_dev *);172void vhost_dev_stop(struct vhost_dev *);
193long vhost_dev_ioctl(struct vhost_dev *, unsigned int ioctl, void __user *argp);173long vhost_dev_ioctl(struct vhost_dev *, unsigned int ioctl, void __user *argp);
@@ -233,6 +213,9 @@ ssize_t vhost_chr_write_iter(struct vhost_dev *dev, · 39 unmodified lines
233 struct iov_iter *from);213 struct iov_iter *from);
234int vhost_init_device_iotlb(struct vhost_dev *d, bool enabled);214int vhost_init_device_iotlb(struct vhost_dev *d, bool enabled);
235 215
216void vhost_iotlb_map_free(struct vhost_iotlb *iotlb,
217 struct vhost_iotlb_map *map);
218
236#define vq_err(vq, fmt, ...) do { \219#define vq_err(vq, fmt, ...) do { \
237 pr_debug(pr_fmt(fmt), ##__VA_ARGS__); \220 pr_debug(pr_fmt(fmt), ##__VA_ARGS__); \
238 if ((vq)->error_ctx) \221 if ((vq)->error_ctx) \
include/linux/vhost_iotlb.h+47−0
@@ -0,0 +1,47 @@
1/* SPDX-License-Identifier: GPL-2.0 */
2#ifndef _LINUX_VHOST_IOTLB_H
3#define _LINUX_VHOST_IOTLB_H
4
5#include <linux/interval_tree_generic.h>
6
7struct vhost_iotlb_map {
8 struct rb_node rb;
9 struct list_head link;
10 u64 start;
11 u64 last;
12 u64 size;
13 u64 addr;
14#define VHOST_MAP_RO 0x1
15#define VHOST_MAP_WO 0x2
16#define VHOST_MAP_RW 0x3
17 u32 perm;
18 u32 flags_padding;
19 u64 __subtree_last;
20};
21
22#define VHOST_IOTLB_FLAG_RETIRE 0x1
23
24struct vhost_iotlb {
25 struct rb_root_cached root;
26 struct list_head list;
27 unsigned int limit;
28 unsigned int nmaps;
29 unsigned int flags;
30};
31
32int vhost_iotlb_add_range(struct vhost_iotlb *iotlb, u64 start, u64 last,
33 u64 addr, unsigned int perm);
34void vhost_iotlb_del_range(struct vhost_iotlb *iotlb, u64 start, u64 last);
35
36struct vhost_iotlb *vhost_iotlb_alloc(unsigned int limit, unsigned int flags);
37void vhost_iotlb_free(struct vhost_iotlb *iotlb);
38void vhost_iotlb_reset(struct vhost_iotlb *iotlb);
39
40struct vhost_iotlb_map *
41vhost_iotlb_itree_first(struct vhost_iotlb *iotlb, u64 start, u64 last);
42struct vhost_iotlb_map *
43vhost_iotlb_itree_next(struct vhost_iotlb_map *map, u64 start, u64 last);
44
45void vhost_iotlb_map_free(struct vhost_iotlb *iotlb,
46 struct vhost_iotlb_map *map);
47#endif