Introduce two ioctls: VDUSE_IOTLB_REG_UMEM and VDUSE_IOTLB_DEREG_UMEM to support registering and de-registering userspace memory for IOVA regions. Now it only supports registering userspace memory for bounce buffer region in virtio-vdpa case.
Unified · Split
| @@ -21,6 +21,8 @@ · 20 unmodified lines | |||
| 21 | #include <linux/uio.h> | 21 | #include <linux/uio.h> |
| 22 | #include <linux/vdpa.h> | 22 | #include <linux/vdpa.h> |
| 23 | #include <linux/nospec.h> | 23 | #include <linux/nospec.h> |
| 24 | #include <linux/vmalloc.h> | ||
| 25 | #include <linux/sched/mm.h> | ||
| 24 | #include <uapi/linux/vduse.h> | 26 | #include <uapi/linux/vduse.h> |
| 25 | #include <uapi/linux/vdpa.h> | 27 | #include <uapi/linux/vdpa.h> |
| 26 | #include <uapi/linux/virtio_config.h> | 28 | #include <uapi/linux/virtio_config.h> |
| @@ -64,6 +66,13 @@ struct vduse_vdpa { · 37 unmodified lines | |||
| 64 | struct vduse_dev *dev; | 66 | struct vduse_dev *dev; |
| 65 | }; | 67 | }; |
| 66 | 68 | ||
| 69 | struct vduse_umem { | ||
| 70 | unsigned long iova; | ||
| 71 | unsigned long npages; | ||
| 72 | struct page **pages; | ||
| 73 | struct mm_struct *mm; | ||
| 74 | }; | ||
| 75 | |||
| 67 | struct vduse_dev { | 76 | struct vduse_dev { |
| 68 | struct vduse_vdpa *vdev; | 77 | struct vduse_vdpa *vdev; |
| 69 | struct device *dev; | 78 | struct device *dev; |
| @@ -95,6 +104,8 @@ struct vduse_dev { · 25 unmodified lines | |||
| 95 | u8 status; | 104 | u8 status; |
| 96 | u32 vq_num; | 105 | u32 vq_num; |
| 97 | u32 vq_align; | 106 | u32 vq_align; |
| 107 | struct vduse_umem *umem; | ||
| 108 | struct mutex mem_lock; | ||
| 98 | }; | 109 | }; |
| 99 | 110 | ||
| 100 | struct vduse_dev_msg { | 111 | struct vduse_dev_msg { |
| @@ -917,6 +928,102 @@ unlock: · 816 unmodified lines | |||
| 917 | return ret; | 928 | return ret; |
| 918 | } | 929 | } |
| 919 | 930 | ||
| 931 | static int vduse_dev_dereg_umem(struct vduse_dev *dev, | ||
| 932 | u64 iova, u64 size) | ||
| 933 | { | ||
| 934 | int ret; | ||
| 935 | |||
| 936 | mutex_lock(&dev->mem_lock); | ||
| 937 | ret = -ENOENT; | ||
| 938 | if (!dev->umem) | ||
| 939 | goto unlock; | ||
| 940 | |||
| 941 | ret = -EINVAL; | ||
| 942 | if (dev->umem->iova != iova || size != dev->domain->bounce_size) | ||
| 943 | goto unlock; | ||
| 944 | |||
| 945 | vduse_domain_remove_user_bounce_pages(dev->domain); | ||
| 946 | unpin_user_pages_dirty_lock(dev->umem->pages, | ||
| 947 | dev->umem->npages, true); | ||
| 948 | atomic64_sub(dev->umem->npages, &dev->umem->mm->pinned_vm); | ||
| 949 | mmdrop(dev->umem->mm); | ||
| 950 | vfree(dev->umem->pages); | ||
| 951 | kfree(dev->umem); | ||
| 952 | dev->umem = NULL; | ||
| 953 | ret = 0; | ||
| 954 | unlock: | ||
| 955 | mutex_unlock(&dev->mem_lock); | ||
| 956 | return ret; | ||
| 957 | } | ||
| 958 | |||
| 959 | static int vduse_dev_reg_umem(struct vduse_dev *dev, | ||
| 960 | u64 iova, u64 uaddr, u64 size) | ||
| 961 | { | ||
| 962 | struct page **page_list = NULL; | ||
| 963 | struct vduse_umem *umem = NULL; | ||
| 964 | long pinned = 0; | ||
| 965 | unsigned long npages, lock_limit; | ||
| 966 | int ret; | ||
| 967 | |||
| 968 | if (!dev->domain->bounce_map || | ||
| 969 | size != dev->domain->bounce_size || | ||
| 970 | iova != 0 || uaddr & ~PAGE_MASK) | ||
| 971 | return -EINVAL; | ||
| 972 | |||
| 973 | mutex_lock(&dev->mem_lock); | ||
| 974 | ret = -EEXIST; | ||
| 975 | if (dev->umem) | ||
| 976 | goto unlock; | ||
| 977 | |||
| 978 | ret = -ENOMEM; | ||
| 979 | npages = size >> PAGE_SHIFT; | ||
| 980 | page_list = __vmalloc(array_size(npages, sizeof(struct page *)), | ||
| 981 | GFP_KERNEL_ACCOUNT); | ||
| 982 | umem = kzalloc(sizeof(*umem), GFP_KERNEL); | ||
| 983 | if (!page_list || !umem) | ||
| 984 | goto unlock; | ||
| 985 | |||
| 986 | mmap_read_lock(current->mm); | ||
| 987 | |||
| 988 | lock_limit = PFN_DOWN(rlimit(RLIMIT_MEMLOCK)); | ||
| 989 | if (npages + atomic64_read(¤t->mm->pinned_vm) > lock_limit) | ||
| 990 | goto out; | ||
| 991 | |||
| 992 | pinned = pin_user_pages(uaddr, npages, FOLL_LONGTERM | FOLL_WRITE, | ||
| 993 | page_list, NULL); | ||
| 994 | if (pinned != npages) { | ||
| 995 | ret = pinned < 0 ? pinned : -ENOMEM; | ||
| 996 | goto out; | ||
| 997 | } | ||
| 998 | |||
| 999 | ret = vduse_domain_add_user_bounce_pages(dev->domain, | ||
| 1000 | page_list, pinned); | ||
| 1001 | if (ret) | ||
| 1002 | goto out; | ||
| 1003 | |||
| 1004 | atomic64_add(npages, ¤t->mm->pinned_vm); | ||
| 1005 | |||
| 1006 | umem->pages = page_list; | ||
| 1007 | umem->npages = pinned; | ||
| 1008 | umem->iova = iova; | ||
| 1009 | umem->mm = current->mm; | ||
| 1010 | mmgrab(current->mm); | ||
| 1011 | |||
| 1012 | dev->umem = umem; | ||
| 1013 | out: | ||
| 1014 | if (ret && pinned > 0) | ||
| 1015 | unpin_user_pages(page_list, pinned); | ||
| 1016 | |||
| 1017 | mmap_read_unlock(current->mm); | ||
| 1018 | unlock: | ||
| 1019 | if (ret) { | ||
| 1020 | vfree(page_list); | ||
| 1021 | kfree(umem); | ||
| 1022 | } | ||
| 1023 | mutex_unlock(&dev->mem_lock); | ||
| 1024 | return ret; | ||
| 1025 | } | ||
| 1026 | |||
| 920 | static long vduse_dev_ioctl(struct file *file, unsigned int cmd, | 1027 | static long vduse_dev_ioctl(struct file *file, unsigned int cmd, |
| 921 | unsigned long arg) | 1028 | unsigned long arg) |
| 922 | { | 1029 | { |
| @@ -1089,6 +1196,38 @@ static long vduse_dev_ioctl(struct file *file, unsigned int cmd, · 166 unmodified lines | |||
| 1089 | ret = vduse_dev_queue_irq_work(dev, &dev->vqs[index].inject); | 1196 | ret = vduse_dev_queue_irq_work(dev, &dev->vqs[index].inject); |
| 1090 | break; | 1197 | break; |
| 1091 | } | 1198 | } |
| 1199 | case VDUSE_IOTLB_REG_UMEM: { | ||
| 1200 | struct vduse_iova_umem umem; | ||
| 1201 | |||
| 1202 | ret = -EFAULT; | ||
| 1203 | if (copy_from_user(&umem, argp, sizeof(umem))) | ||
| 1204 | break; | ||
| 1205 | |||
| 1206 | ret = -EINVAL; | ||
| 1207 | if (!is_mem_zero((const char *)umem.reserved, | ||
| 1208 | sizeof(umem.reserved))) | ||
| 1209 | break; | ||
| 1210 | |||
| 1211 | ret = vduse_dev_reg_umem(dev, umem.iova, | ||
| 1212 | umem.uaddr, umem.size); | ||
| 1213 | break; | ||
| 1214 | } | ||
| 1215 | case VDUSE_IOTLB_DEREG_UMEM: { | ||
| 1216 | struct vduse_iova_umem umem; | ||
| 1217 | |||
| 1218 | ret = -EFAULT; | ||
| 1219 | if (copy_from_user(&umem, argp, sizeof(umem))) | ||
| 1220 | break; | ||
| 1221 | |||
| 1222 | ret = -EINVAL; | ||
| 1223 | if (!is_mem_zero((const char *)umem.reserved, | ||
| 1224 | sizeof(umem.reserved))) | ||
| 1225 | break; | ||
| 1226 | |||
| 1227 | ret = vduse_dev_dereg_umem(dev, umem.iova, | ||
| 1228 | umem.size); | ||
| 1229 | break; | ||
| 1230 | } | ||
| 1092 | default: | 1231 | default: |
| 1093 | ret = -ENOIOCTLCMD; | 1232 | ret = -ENOIOCTLCMD; |
| 1094 | break; | 1233 | break; |
| @@ -1101,6 +1240,7 @@ static int vduse_dev_release(struct inode *inode, struct file *file) · 6 unmodified lines | |||
| 1101 | { | 1240 | { |
| 1102 | struct vduse_dev *dev = file->private_data; | 1241 | struct vduse_dev *dev = file->private_data; |
| 1103 | 1242 | ||
| 1243 | vduse_dev_dereg_umem(dev, 0, dev->domain->bounce_size); | ||
| 1104 | spin_lock(&dev->msg_lock); | 1244 | spin_lock(&dev->msg_lock); |
| 1105 | /* Make sure the inflight messages can processed after reconncection */ | 1245 | /* Make sure the inflight messages can processed after reconncection */ |
| 1106 | list_splice_init(&dev->recv_list, &dev->send_list); | 1246 | list_splice_init(&dev->recv_list, &dev->send_list); |
| @@ -1163,6 +1303,7 @@ static struct vduse_dev *vduse_dev_create(void) · 56 unmodified lines | |||
| 1163 | return NULL; | 1303 | return NULL; |
| 1164 | 1304 | ||
| 1165 | mutex_init(&dev->lock); | 1305 | mutex_init(&dev->lock); |
| 1306 | mutex_init(&dev->mem_lock); | ||
| 1166 | spin_lock_init(&dev->msg_lock); | 1307 | spin_lock_init(&dev->msg_lock); |
| 1167 | INIT_LIST_HEAD(&dev->send_list); | 1308 | INIT_LIST_HEAD(&dev->send_list); |
| 1168 | INIT_LIST_HEAD(&dev->recv_list); | 1309 | INIT_LIST_HEAD(&dev->recv_list); |
| @@ -210,6 +210,29 @@ struct vduse_vq_eventfd { · 209 unmodified lines | |||
| 210 | */ | 210 | */ |
| 211 | #define VDUSE_VQ_INJECT_IRQ _IOW(VDUSE_BASE, 0x17, __u32) | 211 | #define VDUSE_VQ_INJECT_IRQ _IOW(VDUSE_BASE, 0x17, __u32) |
| 212 | 212 | ||
| 213 | /** | ||
| 214 | * struct vduse_iova_umem - userspace memory configuration for one IOVA region | ||
| 215 | * @uaddr: start address of userspace memory, it must be aligned to page size | ||
| 216 | * @iova: start of the IOVA region | ||
| 217 | * @size: size of the IOVA region | ||
| 218 | * @reserved: for future use, needs to be initialized to zero | ||
| 219 | * | ||
| 220 | * Structure used by VDUSE_IOTLB_REG_UMEM and VDUSE_IOTLB_DEREG_UMEM | ||
| 221 | * ioctls to register/de-register userspace memory for IOVA regions | ||
| 222 | */ | ||
| 223 | struct vduse_iova_umem { | ||
| 224 | __u64 uaddr; | ||
| 225 | __u64 iova; | ||
| 226 | __u64 size; | ||
| 227 | __u64 reserved[3]; | ||
| 228 | }; | ||
| 229 | |||
| 230 | /* Register userspace memory for IOVA regions */ | ||
| 231 | #define VDUSE_IOTLB_REG_UMEM _IOW(VDUSE_BASE, 0x18, struct vduse_iova_umem) | ||
| 232 | |||
| 233 | /* De-register the userspace memory. Caller should set iova and size field. */ | ||
| 234 | #define VDUSE_IOTLB_DEREG_UMEM _IOW(VDUSE_BASE, 0x19, struct vduse_iova_umem) | ||
| 235 | |||
| 213 | /* The control messages definition for read(2)/write(2) on /dev/vduse/$NAME */ | 236 | /* The control messages definition for read(2)/write(2) on /dev/vduse/$NAME */ |
| 214 | 237 | ||
| 215 | /** | 238 | /** |