| @@ -0,0 +1,1392 @@ |
| 1 | +// SPDX-License-Identifier: GPL-2.0-only |
| 2 | +/* |
| 3 | + * Copyright (c) 2023, Microsoft Corporation. |
| 4 | + * |
| 5 | + * Author: |
| 6 | + * Roman Kisel <romank@linux.microsoft.com> |
| 7 | + * Saurabh Sengar <ssengar@linux.microsoft.com> |
| 8 | + * Naman Jain <namjain@linux.microsoft.com> |
| 9 | + */ |
| 10 | + |
| 11 | +#include <linux/kernel.h> |
| 12 | +#include <linux/module.h> |
| 13 | +#include <linux/miscdevice.h> |
| 14 | +#include <linux/anon_inodes.h> |
| 15 | +#include <linux/cpuhotplug.h> |
| 16 | +#include <linux/count_zeros.h> |
| 17 | +#include <linux/entry-virt.h> |
| 18 | +#include <linux/eventfd.h> |
| 19 | +#include <linux/poll.h> |
| 20 | +#include <linux/file.h> |
| 21 | +#include <linux/vmalloc.h> |
| 22 | +#include <asm/debugreg.h> |
| 23 | +#include <asm/mshyperv.h> |
| 24 | +#include <trace/events/ipi.h> |
| 25 | +#include <uapi/asm/mtrr.h> |
| 26 | +#include <uapi/linux/mshv.h> |
| 27 | +#include <hyperv/hvhdk.h> |
| 28 | + |
| 29 | +#include "../../kernel/fpu/legacy.h" |
| 30 | +#include "mshv.h" |
| 31 | +#include "mshv_vtl.h" |
| 32 | +#include "hyperv_vmbus.h" |
| 33 | + |
| 34 | +MODULE_AUTHOR("Microsoft"); |
| 35 | +MODULE_LICENSE("GPL"); |
| 36 | +MODULE_DESCRIPTION("Microsoft Hyper-V VTL Driver"); |
| 37 | + |
| 38 | +#define MSHV_ENTRY_REASON_LOWER_VTL_CALL 0x1 |
| 39 | +#define MSHV_ENTRY_REASON_INTERRUPT 0x2 |
| 40 | +#define MSHV_ENTRY_REASON_INTERCEPT 0x3 |
| 41 | + |
| 42 | +#define MSHV_REAL_OFF_SHIFT 16 |
| 43 | +#define MSHV_PG_OFF_CPU_MASK (BIT_ULL(MSHV_REAL_OFF_SHIFT) - 1) |
| 44 | +#define MSHV_RUN_PAGE_OFFSET 0 |
| 45 | +#define MSHV_REG_PAGE_OFFSET 1 |
| 46 | +#define VTL2_VMBUS_SINT_INDEX 7 |
| 47 | + |
| 48 | +static struct device *mem_dev; |
| 49 | + |
| 50 | +static struct tasklet_struct msg_dpc; |
| 51 | +static wait_queue_head_t fd_wait_queue; |
| 52 | +static bool has_message; |
| 53 | +static struct eventfd_ctx *flag_eventfds[HV_EVENT_FLAGS_COUNT]; |
| 54 | +static DEFINE_MUTEX(flag_lock); |
| 55 | +static bool __read_mostly mshv_has_reg_page; |
| 56 | + |
| 57 | +/* hvcall code is of type u16, allocate a bitmap of size (1 << 16) to accommodate it */ |
| 58 | +#define MAX_BITMAP_SIZE ((U16_MAX + 1) / 8) |
| 59 | + |
| 60 | +struct mshv_vtl_hvcall_fd { |
| 61 | + u8 allow_bitmap[MAX_BITMAP_SIZE]; |
| 62 | + bool allow_map_initialized; |
| 63 | + /* |
| 64 | + * Used to protect hvcall setup in IOCTLs |
| 65 | + */ |
| 66 | + struct mutex init_mutex; |
| 67 | + struct miscdevice *dev; |
| 68 | +}; |
| 69 | + |
| 70 | +struct mshv_vtl_poll_file { |
| 71 | + struct file *file; |
| 72 | + wait_queue_entry_t wait; |
| 73 | + wait_queue_head_t *wqh; |
| 74 | + poll_table pt; |
| 75 | + int cpu; |
| 76 | +}; |
| 77 | + |
| 78 | +struct mshv_vtl { |
| 79 | + struct device *module_dev; |
| 80 | + u64 id; |
| 81 | +}; |
| 82 | + |
| 83 | +struct mshv_vtl_per_cpu { |
| 84 | + struct mshv_vtl_run *run; |
| 85 | + struct page *reg_page; |
| 86 | +}; |
| 87 | + |
| 88 | +/* SYNIC_OVERLAY_PAGE_MSR - internal, identical to hv_synic_simp */ |
| 89 | +union hv_synic_overlay_page_msr { |
| 90 | + u64 as_uint64; |
| 91 | + struct { |
| 92 | + u64 enabled: 1; |
| 93 | + u64 reserved: 11; |
| 94 | + u64 pfn: 52; |
| 95 | + } __packed; |
| 96 | +}; |
| 97 | + |
| 98 | +static struct mutex mshv_vtl_poll_file_lock; |
| 99 | +static union hv_register_vsm_page_offsets mshv_vsm_page_offsets; |
| 100 | +static union hv_register_vsm_capabilities mshv_vsm_capabilities; |
| 101 | + |
| 102 | +static DEFINE_PER_CPU(struct mshv_vtl_poll_file, mshv_vtl_poll_file); |
| 103 | +static DEFINE_PER_CPU(unsigned long long, num_vtl0_transitions); |
| 104 | +static DEFINE_PER_CPU(struct mshv_vtl_per_cpu, mshv_vtl_per_cpu); |
| 105 | + |
| 106 | +static const union hv_input_vtl input_vtl_zero; |
| 107 | +static const union hv_input_vtl input_vtl_normal = { |
| 108 | + .use_target_vtl = 1, |
| 109 | +}; |
| 110 | + |
| 111 | +static const struct file_operations mshv_vtl_fops; |
| 112 | + |
| 113 | +static long |
| 114 | +mshv_ioctl_create_vtl(void __user *user_arg, struct device *module_dev) |
| 115 | +{ |
| 116 | + struct mshv_vtl *vtl; |
| 117 | + struct file *file; |
| 118 | + int fd; |
| 119 | + |
| 120 | + vtl = kzalloc(sizeof(*vtl), GFP_KERNEL); |
| 121 | + if (!vtl) |
| 122 | + return -ENOMEM; |
| 123 | + |
| 124 | + fd = get_unused_fd_flags(O_CLOEXEC); |
| 125 | + if (fd < 0) { |
| 126 | + kfree(vtl); |
| 127 | + return fd; |
| 128 | + } |
| 129 | + file = anon_inode_getfile("mshv_vtl", &mshv_vtl_fops, |
| 130 | + vtl, O_RDWR); |
| 131 | + if (IS_ERR(file)) { |
| 132 | + kfree(vtl); |
| 133 | + return PTR_ERR(file); |
| 134 | + } |
| 135 | + vtl->module_dev = module_dev; |
| 136 | + fd_install(fd, file); |
| 137 | + |
| 138 | + return fd; |
| 139 | +} |
| 140 | + |
| 141 | +static long |
| 142 | +mshv_ioctl_check_extension(void __user *user_arg) |
| 143 | +{ |
| 144 | + u32 arg; |
| 145 | + |
| 146 | + if (copy_from_user(&arg, user_arg, sizeof(arg))) |
| 147 | + return -EFAULT; |
| 148 | + |
| 149 | + switch (arg) { |
| 150 | + case MSHV_CAP_CORE_API_STABLE: |
| 151 | + return 0; |
| 152 | + case MSHV_CAP_REGISTER_PAGE: |
| 153 | + return mshv_has_reg_page; |
| 154 | + case MSHV_CAP_VTL_RETURN_ACTION: |
| 155 | + return mshv_vsm_capabilities.return_action_available; |
| 156 | + case MSHV_CAP_DR6_SHARED: |
| 157 | + return mshv_vsm_capabilities.dr6_shared; |
| 158 | + } |
| 159 | + |
| 160 | + return -EOPNOTSUPP; |
| 161 | +} |
| 162 | + |
| 163 | +static long |
| 164 | +mshv_dev_ioctl(struct file *filp, unsigned int ioctl, unsigned long arg) |
| 165 | +{ |
| 166 | + struct miscdevice *misc = filp->private_data; |
| 167 | + |
| 168 | + switch (ioctl) { |
| 169 | + case MSHV_CHECK_EXTENSION: |
| 170 | + return mshv_ioctl_check_extension((void __user *)arg); |
| 171 | + case MSHV_CREATE_VTL: |
| 172 | + return mshv_ioctl_create_vtl((void __user *)arg, misc->this_device); |
| 173 | + } |
| 174 | + |
| 175 | + return -ENOTTY; |
| 176 | +} |
| 177 | + |
| 178 | +static const struct file_operations mshv_dev_fops = { |
| 179 | + .owner = THIS_MODULE, |
| 180 | + .unlocked_ioctl = mshv_dev_ioctl, |
| 181 | + .llseek = noop_llseek, |
| 182 | +}; |
| 183 | + |
| 184 | +static struct miscdevice mshv_dev = { |
| 185 | + .minor = MISC_DYNAMIC_MINOR, |
| 186 | + .name = "mshv", |
| 187 | + .fops = &mshv_dev_fops, |
| 188 | + .mode = 0600, |
| 189 | +}; |
| 190 | + |
| 191 | +static struct mshv_vtl_run *mshv_vtl_this_run(void) |
| 192 | +{ |
| 193 | + return *this_cpu_ptr(&mshv_vtl_per_cpu.run); |
| 194 | +} |
| 195 | + |
| 196 | +static struct mshv_vtl_run *mshv_vtl_cpu_run(int cpu) |
| 197 | +{ |
| 198 | + return *per_cpu_ptr(&mshv_vtl_per_cpu.run, cpu); |
| 199 | +} |
| 200 | + |
| 201 | +static struct page *mshv_vtl_cpu_reg_page(int cpu) |
| 202 | +{ |
| 203 | + return *per_cpu_ptr(&mshv_vtl_per_cpu.reg_page, cpu); |
| 204 | +} |
| 205 | + |
| 206 | +static void mshv_vtl_configure_reg_page(struct mshv_vtl_per_cpu *per_cpu) |
| 207 | +{ |
| 208 | + struct hv_register_assoc reg_assoc = {}; |
| 209 | + union hv_synic_overlay_page_msr overlay = {}; |
| 210 | + struct page *reg_page; |
| 211 | + |
| 212 | + reg_page = alloc_page(GFP_KERNEL | __GFP_ZERO | __GFP_RETRY_MAYFAIL); |
| 213 | + if (!reg_page) { |
| 214 | + WARN(1, "failed to allocate register page\n"); |
| 215 | + return; |
| 216 | + } |
| 217 | + |
| 218 | + overlay.enabled = 1; |
| 219 | + overlay.pfn = page_to_hvpfn(reg_page); |
| 220 | + reg_assoc.name = HV_X64_REGISTER_REG_PAGE; |
| 221 | + reg_assoc.value.reg64 = overlay.as_uint64; |
| 222 | + |
| 223 | + if (hv_call_set_vp_registers(HV_VP_INDEX_SELF, HV_PARTITION_ID_SELF, |
| 224 | + 1, input_vtl_zero, ®_assoc)) { |
| 225 | + WARN(1, "failed to setup register page\n"); |
| 226 | + __free_page(reg_page); |
| 227 | + return; |
| 228 | + } |
| 229 | + |
| 230 | + per_cpu->reg_page = reg_page; |
| 231 | + mshv_has_reg_page = true; |
| 232 | +} |
| 233 | + |
| 234 | +static void mshv_vtl_synic_enable_regs(unsigned int cpu) |
| 235 | +{ |
| 236 | + union hv_synic_sint sint; |
| 237 | + |
| 238 | + sint.as_uint64 = 0; |
| 239 | + sint.vector = HYPERVISOR_CALLBACK_VECTOR; |
| 240 | + sint.masked = false; |
| 241 | + sint.auto_eoi = hv_recommend_using_aeoi(); |
| 242 | + |
| 243 | + /* Enable intercepts */ |
| 244 | + if (!mshv_vsm_capabilities.intercept_page_available) |
| 245 | + hv_set_msr(HV_MSR_SINT0 + HV_SYNIC_INTERCEPTION_SINT_INDEX, |
| 246 | + sint.as_uint64); |
| 247 | + |
| 248 | + /* VTL2 Host VSP SINT is (un)masked when the user mode requests that */ |
| 249 | +} |
| 250 | + |
| 251 | +static int mshv_vtl_get_vsm_regs(void) |
| 252 | +{ |
| 253 | + struct hv_register_assoc registers[2]; |
| 254 | + int ret, count = 2; |
| 255 | + |
| 256 | + registers[0].name = HV_REGISTER_VSM_CODE_PAGE_OFFSETS; |
| 257 | + registers[1].name = HV_REGISTER_VSM_CAPABILITIES; |
| 258 | + |
| 259 | + ret = hv_call_get_vp_registers(HV_VP_INDEX_SELF, HV_PARTITION_ID_SELF, |
| 260 | + count, input_vtl_zero, registers); |
| 261 | + if (ret) |
| 262 | + return ret; |
| 263 | + |
| 264 | + mshv_vsm_page_offsets.as_uint64 = registers[0].value.reg64; |
| 265 | + mshv_vsm_capabilities.as_uint64 = registers[1].value.reg64; |
| 266 | + |
| 267 | + return ret; |
| 268 | +} |
| 269 | + |
| 270 | +static int mshv_vtl_configure_vsm_partition(struct device *dev) |
| 271 | +{ |
| 272 | + union hv_register_vsm_partition_config config; |
| 273 | + struct hv_register_assoc reg_assoc; |
| 274 | + |
| 275 | + config.as_uint64 = 0; |
| 276 | + config.default_vtl_protection_mask = HV_MAP_GPA_PERMISSIONS_MASK; |
| 277 | + config.enable_vtl_protection = 1; |
| 278 | + config.zero_memory_on_reset = 1; |
| 279 | + config.intercept_vp_startup = 1; |
| 280 | + config.intercept_cpuid_unimplemented = 1; |
| 281 | + |
| 282 | + if (mshv_vsm_capabilities.intercept_page_available) { |
| 283 | + dev_dbg(dev, "using intercept page\n"); |
| 284 | + config.intercept_page = 1; |
| 285 | + } |
| 286 | + |
| 287 | + reg_assoc.name = HV_REGISTER_VSM_PARTITION_CONFIG; |
| 288 | + reg_assoc.value.reg64 = config.as_uint64; |
| 289 | + |
| 290 | + return hv_call_set_vp_registers(HV_VP_INDEX_SELF, HV_PARTITION_ID_SELF, |
| 291 | + 1, input_vtl_zero, ®_assoc); |
| 292 | +} |
| 293 | + |
| 294 | +static void mshv_vtl_vmbus_isr(void) |
| 295 | +{ |
| 296 | + struct hv_per_cpu_context *per_cpu; |
| 297 | + struct hv_message *msg; |
| 298 | + u32 message_type; |
| 299 | + union hv_synic_event_flags *event_flags; |
| 300 | + struct eventfd_ctx *eventfd; |
| 301 | + u16 i; |
| 302 | + |
| 303 | + per_cpu = this_cpu_ptr(hv_context.cpu_context); |
| 304 | + if (smp_processor_id() == 0) { |
| 305 | + msg = (struct hv_message *)per_cpu->hyp_synic_message_page + VTL2_VMBUS_SINT_INDEX; |
| 306 | + message_type = READ_ONCE(msg->header.message_type); |
| 307 | + if (message_type != HVMSG_NONE) |
| 308 | + tasklet_schedule(&msg_dpc); |
| 309 | + } |
| 310 | + |
| 311 | + event_flags = (union hv_synic_event_flags *)per_cpu->hyp_synic_event_page + |
| 312 | + VTL2_VMBUS_SINT_INDEX; |
| 313 | + for_each_set_bit(i, event_flags->flags, HV_EVENT_FLAGS_COUNT) { |
| 314 | + if (!sync_test_and_clear_bit(i, event_flags->flags)) |
| 315 | + continue; |
| 316 | + rcu_read_lock(); |
| 317 | + eventfd = READ_ONCE(flag_eventfds[i]); |
| 318 | + if (eventfd) |
| 319 | + eventfd_signal(eventfd); |
| 320 | + rcu_read_unlock(); |
| 321 | + } |
| 322 | + |
| 323 | + vmbus_isr(); |
| 324 | +} |
| 325 | + |
| 326 | +static int mshv_vtl_alloc_context(unsigned int cpu) |
| 327 | +{ |
| 328 | + struct mshv_vtl_per_cpu *per_cpu = this_cpu_ptr(&mshv_vtl_per_cpu); |
| 329 | + |
| 330 | + per_cpu->run = (struct mshv_vtl_run *)__get_free_page(GFP_KERNEL | __GFP_ZERO); |
| 331 | + if (!per_cpu->run) |
| 332 | + return -ENOMEM; |
| 333 | + |
| 334 | + if (mshv_vsm_capabilities.intercept_page_available) |
| 335 | + mshv_vtl_configure_reg_page(per_cpu); |
| 336 | + |
| 337 | + mshv_vtl_synic_enable_regs(cpu); |
| 338 | + |
| 339 | + return 0; |
| 340 | +} |
| 341 | + |
| 342 | +static int mshv_vtl_cpuhp_online; |
| 343 | + |
| 344 | +static int hv_vtl_setup_synic(void) |
| 345 | +{ |
| 346 | + int ret; |
| 347 | + |
| 348 | + /* Use our isr to first filter out packets destined for userspace */ |
| 349 | + hv_setup_vmbus_handler(mshv_vtl_vmbus_isr); |
| 350 | + |
| 351 | + ret = cpuhp_setup_state(CPUHP_AP_ONLINE_DYN, "hyperv/vtl:online", |
| 352 | + mshv_vtl_alloc_context, NULL); |
| 353 | + if (ret < 0) { |
| 354 | + hv_setup_vmbus_handler(vmbus_isr); |
| 355 | + return ret; |
| 356 | + } |
| 357 | + |
| 358 | + mshv_vtl_cpuhp_online = ret; |
| 359 | + |
| 360 | + return 0; |
| 361 | +} |
| 362 | + |
| 363 | +static void hv_vtl_remove_synic(void) |
| 364 | +{ |
| 365 | + cpuhp_remove_state(mshv_vtl_cpuhp_online); |
| 366 | + hv_setup_vmbus_handler(vmbus_isr); |
| 367 | +} |
| 368 | + |
| 369 | +static int vtl_get_vp_register(struct hv_register_assoc *reg) |
| 370 | +{ |
| 371 | + return hv_call_get_vp_registers(HV_VP_INDEX_SELF, HV_PARTITION_ID_SELF, |
| 372 | + 1, input_vtl_normal, reg); |
| 373 | +} |
| 374 | + |
| 375 | +static int vtl_set_vp_register(struct hv_register_assoc *reg) |
| 376 | +{ |
| 377 | + return hv_call_set_vp_registers(HV_VP_INDEX_SELF, HV_PARTITION_ID_SELF, |
| 378 | + 1, input_vtl_normal, reg); |
| 379 | +} |
| 380 | + |
| 381 | +static int mshv_vtl_ioctl_add_vtl0_mem(struct mshv_vtl *vtl, void __user *arg) |
| 382 | +{ |
| 383 | + struct mshv_vtl_ram_disposition vtl0_mem; |
| 384 | + struct dev_pagemap *pgmap; |
| 385 | + void *addr; |
| 386 | + |
| 387 | + if (copy_from_user(&vtl0_mem, arg, sizeof(vtl0_mem))) |
| 388 | + return -EFAULT; |
| 389 | + /* vtl0_mem.last_pfn is excluded in the pagemap range for VTL0 as per design */ |
| 390 | + if (vtl0_mem.last_pfn <= vtl0_mem.start_pfn) { |
| 391 | + dev_err(vtl->module_dev, "range start pfn (%llx) > end pfn (%llx)\n", |
| 392 | + vtl0_mem.start_pfn, vtl0_mem.last_pfn); |
| 393 | + return -EFAULT; |
| 394 | + } |
| 395 | + |
| 396 | + pgmap = kzalloc(sizeof(*pgmap), GFP_KERNEL); |
| 397 | + if (!pgmap) |
| 398 | + return -ENOMEM; |
| 399 | + |
| 400 | + pgmap->ranges[0].start = PFN_PHYS(vtl0_mem.start_pfn); |
| 401 | + pgmap->ranges[0].end = PFN_PHYS(vtl0_mem.last_pfn) - 1; |
| 402 | + pgmap->nr_range = 1; |
| 403 | + pgmap->type = MEMORY_DEVICE_GENERIC; |
| 404 | + |
| 405 | + /* |
| 406 | + * Determine the highest page order that can be used for the given memory range. |
| 407 | + * This works best when the range is aligned; i.e. both the start and the length. |
| 408 | + */ |
| 409 | + pgmap->vmemmap_shift = count_trailing_zeros(vtl0_mem.start_pfn | vtl0_mem.last_pfn); |
| 410 | + dev_dbg(vtl->module_dev, |
| 411 | + "Add VTL0 memory: start: 0x%llx, end_pfn: 0x%llx, page order: %lu\n", |
| 412 | + vtl0_mem.start_pfn, vtl0_mem.last_pfn, pgmap->vmemmap_shift); |
| 413 | + |
| 414 | + addr = devm_memremap_pages(mem_dev, pgmap); |
| 415 | + if (IS_ERR(addr)) { |
| 416 | + dev_err(vtl->module_dev, "devm_memremap_pages error: %ld\n", PTR_ERR(addr)); |
| 417 | + kfree(pgmap); |
| 418 | + return -EFAULT; |
| 419 | + } |
| 420 | + |
| 421 | + /* Don't free pgmap, since it has to stick around until the memory |
| 422 | + * is unmapped, which will never happen as there is no scenario |
| 423 | + * where VTL0 can be released/shutdown without bringing down VTL2. |
| 424 | + */ |
| 425 | + return 0; |
| 426 | +} |
| 427 | + |
| 428 | +static void mshv_vtl_cancel(int cpu) |
| 429 | +{ |
| 430 | + int here = get_cpu(); |
| 431 | + |
| 432 | + if (here != cpu) { |
| 433 | + if (!xchg_relaxed(&mshv_vtl_cpu_run(cpu)->cancel, 1)) |
| 434 | + smp_send_reschedule(cpu); |
| 435 | + } else { |
| 436 | + WRITE_ONCE(mshv_vtl_this_run()->cancel, 1); |
| 437 | + } |
| 438 | + put_cpu(); |
| 439 | +} |
| 440 | + |
| 441 | +static int mshv_vtl_poll_file_wake(wait_queue_entry_t *wait, unsigned int mode, int sync, void *key) |
| 442 | +{ |
| 443 | + struct mshv_vtl_poll_file *poll_file = container_of(wait, struct mshv_vtl_poll_file, wait); |
| 444 | + |
| 445 | + mshv_vtl_cancel(poll_file->cpu); |
| 446 | + |
| 447 | + return 0; |
| 448 | +} |
| 449 | + |
| 450 | +static void mshv_vtl_ptable_queue_proc(struct file *file, wait_queue_head_t *wqh, poll_table *pt) |
| 451 | +{ |
| 452 | + struct mshv_vtl_poll_file *poll_file = container_of(pt, struct mshv_vtl_poll_file, pt); |
| 453 | + |
| 454 | + WARN_ON(poll_file->wqh); |
| 455 | + poll_file->wqh = wqh; |
| 456 | + add_wait_queue(wqh, &poll_file->wait); |
| 457 | +} |
| 458 | + |
| 459 | +static int mshv_vtl_ioctl_set_poll_file(struct mshv_vtl_set_poll_file __user *user_input) |
| 460 | +{ |
| 461 | + struct file *file, *old_file; |
| 462 | + struct mshv_vtl_poll_file *poll_file; |
| 463 | + struct mshv_vtl_set_poll_file input; |
| 464 | + |
| 465 | + if (copy_from_user(&input, user_input, sizeof(input))) |
| 466 | + return -EFAULT; |
| 467 | + |
| 468 | + if (input.cpu >= num_possible_cpus() || !cpu_online(input.cpu)) |
| 469 | + return -EINVAL; |
| 470 | + /* |
| 471 | + * CPU Hotplug is not supported in VTL2 in OpenHCL, where this kernel driver exists. |
| 472 | + * CPU is expected to remain online after above cpu_online() check. |
| 473 | + */ |
| 474 | + |
| 475 | + file = NULL; |
| 476 | + file = fget(input.fd); |
| 477 | + if (!file) |
| 478 | + return -EBADFD; |
| 479 | + |
| 480 | + poll_file = per_cpu_ptr(&mshv_vtl_poll_file, READ_ONCE(input.cpu)); |
| 481 | + if (!poll_file) |
| 482 | + return -EINVAL; |
| 483 | + |
| 484 | + mutex_lock(&mshv_vtl_poll_file_lock); |
| 485 | + |
| 486 | + if (poll_file->wqh) |
| 487 | + remove_wait_queue(poll_file->wqh, &poll_file->wait); |
| 488 | + poll_file->wqh = NULL; |
| 489 | + |
| 490 | + old_file = poll_file->file; |
| 491 | + poll_file->file = file; |
| 492 | + poll_file->cpu = input.cpu; |
| 493 | + |
| 494 | + if (file) { |
| 495 | + init_waitqueue_func_entry(&poll_file->wait, mshv_vtl_poll_file_wake); |
| 496 | + init_poll_funcptr(&poll_file->pt, mshv_vtl_ptable_queue_proc); |
| 497 | + vfs_poll(file, &poll_file->pt); |
| 498 | + } |
| 499 | + |
| 500 | + mutex_unlock(&mshv_vtl_poll_file_lock); |
| 501 | + |
| 502 | + if (old_file) |
| 503 | + fput(old_file); |
| 504 | + |
| 505 | + return 0; |
| 506 | +} |
| 507 | + |
| 508 | +/* Static table mapping register names to their corresponding actions */ |
| 509 | +static const struct { |
| 510 | + enum hv_register_name reg_name; |
| 511 | + int debug_reg_num; /* -1 if not a debug register */ |
| 512 | + u32 msr_addr; /* 0 if not an MSR */ |
| 513 | +} reg_table[] = { |
| 514 | + /* Debug registers */ |
| 515 | + {HV_X64_REGISTER_DR0, 0, 0}, |
| 516 | + {HV_X64_REGISTER_DR1, 1, 0}, |
| 517 | + {HV_X64_REGISTER_DR2, 2, 0}, |
| 518 | + {HV_X64_REGISTER_DR3, 3, 0}, |
| 519 | + {HV_X64_REGISTER_DR6, 6, 0}, |
| 520 | + /* MTRR MSRs */ |
| 521 | + {HV_X64_REGISTER_MSR_MTRR_CAP, -1, MSR_MTRRcap}, |
| 522 | + {HV_X64_REGISTER_MSR_MTRR_DEF_TYPE, -1, MSR_MTRRdefType}, |
| 523 | + {HV_X64_REGISTER_MSR_MTRR_PHYS_BASE0, -1, MTRRphysBase_MSR(0)}, |
| 524 | + {HV_X64_REGISTER_MSR_MTRR_PHYS_BASE1, -1, MTRRphysBase_MSR(1)}, |
| 525 | + {HV_X64_REGISTER_MSR_MTRR_PHYS_BASE2, -1, MTRRphysBase_MSR(2)}, |
| 526 | + {HV_X64_REGISTER_MSR_MTRR_PHYS_BASE3, -1, MTRRphysBase_MSR(3)}, |
| 527 | + {HV_X64_REGISTER_MSR_MTRR_PHYS_BASE4, -1, MTRRphysBase_MSR(4)}, |
| 528 | + {HV_X64_REGISTER_MSR_MTRR_PHYS_BASE5, -1, MTRRphysBase_MSR(5)}, |
| 529 | + {HV_X64_REGISTER_MSR_MTRR_PHYS_BASE6, -1, MTRRphysBase_MSR(6)}, |
| 530 | + {HV_X64_REGISTER_MSR_MTRR_PHYS_BASE7, -1, MTRRphysBase_MSR(7)}, |
| 531 | + {HV_X64_REGISTER_MSR_MTRR_PHYS_BASE8, -1, MTRRphysBase_MSR(8)}, |
| 532 | + {HV_X64_REGISTER_MSR_MTRR_PHYS_BASE9, -1, MTRRphysBase_MSR(9)}, |
| 533 | + {HV_X64_REGISTER_MSR_MTRR_PHYS_BASEA, -1, MTRRphysBase_MSR(0xa)}, |
| 534 | + {HV_X64_REGISTER_MSR_MTRR_PHYS_BASEB, -1, MTRRphysBase_MSR(0xb)}, |
| 535 | + {HV_X64_REGISTER_MSR_MTRR_PHYS_BASEC, -1, MTRRphysBase_MSR(0xc)}, |
| 536 | + {HV_X64_REGISTER_MSR_MTRR_PHYS_BASED, -1, MTRRphysBase_MSR(0xd)}, |
| 537 | + {HV_X64_REGISTER_MSR_MTRR_PHYS_BASEE, -1, MTRRphysBase_MSR(0xe)}, |
| 538 | + {HV_X64_REGISTER_MSR_MTRR_PHYS_BASEF, -1, MTRRphysBase_MSR(0xf)}, |
| 539 | + {HV_X64_REGISTER_MSR_MTRR_PHYS_MASK0, -1, MTRRphysMask_MSR(0)}, |
| 540 | + {HV_X64_REGISTER_MSR_MTRR_PHYS_MASK1, -1, MTRRphysMask_MSR(1)}, |
| 541 | + {HV_X64_REGISTER_MSR_MTRR_PHYS_MASK2, -1, MTRRphysMask_MSR(2)}, |
| 542 | + {HV_X64_REGISTER_MSR_MTRR_PHYS_MASK3, -1, MTRRphysMask_MSR(3)}, |
| 543 | + {HV_X64_REGISTER_MSR_MTRR_PHYS_MASK4, -1, MTRRphysMask_MSR(4)}, |
| 544 | + {HV_X64_REGISTER_MSR_MTRR_PHYS_MASK5, -1, MTRRphysMask_MSR(5)}, |
| 545 | + {HV_X64_REGISTER_MSR_MTRR_PHYS_MASK6, -1, MTRRphysMask_MSR(6)}, |
| 546 | + {HV_X64_REGISTER_MSR_MTRR_PHYS_MASK7, -1, MTRRphysMask_MSR(7)}, |
| 547 | + {HV_X64_REGISTER_MSR_MTRR_PHYS_MASK8, -1, MTRRphysMask_MSR(8)}, |
| 548 | + {HV_X64_REGISTER_MSR_MTRR_PHYS_MASK9, -1, MTRRphysMask_MSR(9)}, |
| 549 | + {HV_X64_REGISTER_MSR_MTRR_PHYS_MASKA, -1, MTRRphysMask_MSR(0xa)}, |
| 550 | + {HV_X64_REGISTER_MSR_MTRR_PHYS_MASKB, -1, MTRRphysMask_MSR(0xb)}, |
| 551 | + {HV_X64_REGISTER_MSR_MTRR_PHYS_MASKC, -1, MTRRphysMask_MSR(0xc)}, |
| 552 | + {HV_X64_REGISTER_MSR_MTRR_PHYS_MASKD, -1, MTRRphysMask_MSR(0xd)}, |
| 553 | + {HV_X64_REGISTER_MSR_MTRR_PHYS_MASKE, -1, MTRRphysMask_MSR(0xe)}, |
| 554 | + {HV_X64_REGISTER_MSR_MTRR_PHYS_MASKF, -1, MTRRphysMask_MSR(0xf)}, |
| 555 | + {HV_X64_REGISTER_MSR_MTRR_FIX64K00000, -1, MSR_MTRRfix64K_00000}, |
| 556 | + {HV_X64_REGISTER_MSR_MTRR_FIX16K80000, -1, MSR_MTRRfix16K_80000}, |
| 557 | + {HV_X64_REGISTER_MSR_MTRR_FIX16KA0000, -1, MSR_MTRRfix16K_A0000}, |
| 558 | + {HV_X64_REGISTER_MSR_MTRR_FIX4KC0000, -1, MSR_MTRRfix4K_C0000}, |
| 559 | + {HV_X64_REGISTER_MSR_MTRR_FIX4KC8000, -1, MSR_MTRRfix4K_C8000}, |
| 560 | + {HV_X64_REGISTER_MSR_MTRR_FIX4KD0000, -1, MSR_MTRRfix4K_D0000}, |
| 561 | + {HV_X64_REGISTER_MSR_MTRR_FIX4KD8000, -1, MSR_MTRRfix4K_D8000}, |
| 562 | + {HV_X64_REGISTER_MSR_MTRR_FIX4KE0000, -1, MSR_MTRRfix4K_E0000}, |
| 563 | + {HV_X64_REGISTER_MSR_MTRR_FIX4KE8000, -1, MSR_MTRRfix4K_E8000}, |
| 564 | + {HV_X64_REGISTER_MSR_MTRR_FIX4KF0000, -1, MSR_MTRRfix4K_F0000}, |
| 565 | + {HV_X64_REGISTER_MSR_MTRR_FIX4KF8000, -1, MSR_MTRRfix4K_F8000}, |
| 566 | +}; |
| 567 | + |
| 568 | +static int mshv_vtl_get_set_reg(struct hv_register_assoc *regs, bool set) |
| 569 | +{ |
| 570 | + u64 *reg64; |
| 571 | + enum hv_register_name gpr_name; |
| 572 | + int i; |
| 573 | + |
| 574 | + gpr_name = regs->name; |
| 575 | + reg64 = ®s->value.reg64; |
| 576 | + |
| 577 | + /* Search for the register in the table */ |
| 578 | + for (i = 0; i < ARRAY_SIZE(reg_table); i++) { |
| 579 | + if (reg_table[i].reg_name != gpr_name) |
| 580 | + continue; |
| 581 | + if (reg_table[i].debug_reg_num != -1) { |
| 582 | + /* Handle debug registers */ |
| 583 | + if (gpr_name == HV_X64_REGISTER_DR6 && |
| 584 | + !mshv_vsm_capabilities.dr6_shared) |
| 585 | + goto hypercall; |
| 586 | + if (set) |
| 587 | + native_set_debugreg(reg_table[i].debug_reg_num, *reg64); |
| 588 | + else |
| 589 | + *reg64 = native_get_debugreg(reg_table[i].debug_reg_num); |
| 590 | + } else { |
| 591 | + /* Handle MSRs */ |
| 592 | + if (set) |
| 593 | + wrmsrl(reg_table[i].msr_addr, *reg64); |
| 594 | + else |
| 595 | + rdmsrl(reg_table[i].msr_addr, *reg64); |
| 596 | + } |
| 597 | + return 0; |
| 598 | + } |
| 599 | + |
| 600 | +hypercall: |
| 601 | + return 1; |
| 602 | +} |
| 603 | + |
| 604 | +static void mshv_vtl_return(struct mshv_vtl_cpu_context *vtl0) |
| 605 | +{ |
| 606 | + struct hv_vp_assist_page *hvp; |
| 607 | + |
| 608 | + hvp = hv_vp_assist_page[smp_processor_id()]; |
| 609 | + |
| 610 | + /* |
| 611 | + * Process signal event direct set in the run page, if any. |
| 612 | + */ |
| 613 | + if (mshv_vsm_capabilities.return_action_available) { |
| 614 | + u32 offset = READ_ONCE(mshv_vtl_this_run()->vtl_ret_action_size); |
| 615 | + |
| 616 | + WRITE_ONCE(mshv_vtl_this_run()->vtl_ret_action_size, 0); |
| 617 | + |
| 618 | + /* |
| 619 | + * Hypervisor will take care of clearing out the actions |
| 620 | + * set in the assist page. |
| 621 | + */ |
| 622 | + memcpy(hvp->vtl_ret_actions, |
| 623 | + mshv_vtl_this_run()->vtl_ret_actions, |
| 624 | + min_t(u32, offset, sizeof(hvp->vtl_ret_actions))); |
| 625 | + } |
| 626 | + |
| 627 | + mshv_vtl_return_call(vtl0); |
| 628 | +} |
| 629 | + |
| 630 | +static bool mshv_vtl_process_intercept(void) |
| 631 | +{ |
| 632 | + struct hv_per_cpu_context *mshv_cpu; |
| 633 | + void *synic_message_page; |
| 634 | + struct hv_message *msg; |
| 635 | + u32 message_type; |
| 636 | + |
| 637 | + mshv_cpu = this_cpu_ptr(hv_context.cpu_context); |
| 638 | + synic_message_page = mshv_cpu->hyp_synic_message_page; |
| 639 | + if (unlikely(!synic_message_page)) |
| 640 | + return true; |
| 641 | + |
| 642 | + msg = (struct hv_message *)synic_message_page + HV_SYNIC_INTERCEPTION_SINT_INDEX; |
| 643 | + message_type = READ_ONCE(msg->header.message_type); |
| 644 | + if (message_type == HVMSG_NONE) |
| 645 | + return true; |
| 646 | + |
| 647 | + memcpy(mshv_vtl_this_run()->exit_message, msg, sizeof(*msg)); |
| 648 | + vmbus_signal_eom(msg, message_type); |
| 649 | + |
| 650 | + return false; |
| 651 | +} |
| 652 | + |
| 653 | +static int mshv_vtl_ioctl_return_to_lower_vtl(void) |
| 654 | +{ |
| 655 | + preempt_disable(); |
| 656 | + for (;;) { |
| 657 | + unsigned long irq_flags; |
| 658 | + struct hv_vp_assist_page *hvp; |
| 659 | + int ret; |
| 660 | + |
| 661 | + if (__xfer_to_guest_mode_work_pending()) { |
| 662 | + preempt_enable(); |
| 663 | + ret = xfer_to_guest_mode_handle_work(); |
| 664 | + if (ret) |
| 665 | + return ret; |
| 666 | + preempt_disable(); |
| 667 | + } |
| 668 | + |
| 669 | + local_irq_save(irq_flags); |
| 670 | + if (READ_ONCE(mshv_vtl_this_run()->cancel)) { |
| 671 | + local_irq_restore(irq_flags); |
| 672 | + preempt_enable(); |
| 673 | + return -EINTR; |
| 674 | + } |
| 675 | + |
| 676 | + mshv_vtl_return(&mshv_vtl_this_run()->cpu_context); |
| 677 | + local_irq_restore(irq_flags); |
| 678 | + |
| 679 | + hvp = hv_vp_assist_page[smp_processor_id()]; |
| 680 | + this_cpu_inc(num_vtl0_transitions); |
| 681 | + switch (hvp->vtl_entry_reason) { |
| 682 | + case MSHV_ENTRY_REASON_INTERRUPT: |
| 683 | + if (!mshv_vsm_capabilities.intercept_page_available && |
| 684 | + likely(!mshv_vtl_process_intercept())) |
| 685 | + goto done; |
| 686 | + break; |
| 687 | + |
| 688 | + case MSHV_ENTRY_REASON_INTERCEPT: |
| 689 | + WARN_ON(!mshv_vsm_capabilities.intercept_page_available); |
| 690 | + memcpy(mshv_vtl_this_run()->exit_message, hvp->intercept_message, |
| 691 | + sizeof(hvp->intercept_message)); |
| 692 | + goto done; |
| 693 | + |
| 694 | + default: |
| 695 | + panic("unknown entry reason: %d", hvp->vtl_entry_reason); |
| 696 | + } |
| 697 | + } |
| 698 | + |
| 699 | +done: |
| 700 | + preempt_enable(); |
| 701 | + |
| 702 | + return 0; |
| 703 | +} |
| 704 | + |
| 705 | +static long |
| 706 | +mshv_vtl_ioctl_get_regs(void __user *user_args) |
| 707 | +{ |
| 708 | + struct mshv_vp_registers args; |
| 709 | + struct hv_register_assoc reg; |
| 710 | + long ret; |
| 711 | + |
| 712 | + if (copy_from_user(&args, user_args, sizeof(args))) |
| 713 | + return -EFAULT; |
| 714 | + |
| 715 | + /* This IOCTL supports processing only one register at a time. */ |
| 716 | + if (args.count != 1) |
| 717 | + return -EINVAL; |
| 718 | + |
| 719 | + if (copy_from_user(®, (void __user *)args.regs_ptr, |
| 720 | + sizeof(reg))) |
| 721 | + return -EFAULT; |
| 722 | + |
| 723 | + ret = mshv_vtl_get_set_reg(®, false); |
| 724 | + if (!ret) |
| 725 | + goto copy_args; /* No need of hypercall */ |
| 726 | + ret = vtl_get_vp_register(®); |
| 727 | + if (ret) |
| 728 | + return ret; |
| 729 | + |
| 730 | +copy_args: |
| 731 | + if (copy_to_user((void __user *)args.regs_ptr, ®, sizeof(reg))) |
| 732 | + ret = -EFAULT; |
| 733 | + |
| 734 | + return ret; |
| 735 | +} |
| 736 | + |
| 737 | +static long |
| 738 | +mshv_vtl_ioctl_set_regs(void __user *user_args) |
| 739 | +{ |
| 740 | + struct mshv_vp_registers args; |
| 741 | + struct hv_register_assoc reg; |
| 742 | + long ret; |
| 743 | + |
| 744 | + if (copy_from_user(&args, user_args, sizeof(args))) |
| 745 | + return -EFAULT; |
| 746 | + |
| 747 | + /* This IOCTL supports processing only one register at a time. */ |
| 748 | + if (args.count != 1) |
| 749 | + return -EINVAL; |
| 750 | + |
| 751 | + if (copy_from_user(®, (void __user *)args.regs_ptr, sizeof(reg))) |
| 752 | + return -EFAULT; |
| 753 | + |
| 754 | + ret = mshv_vtl_get_set_reg(®, true); |
| 755 | + if (!ret) |
| 756 | + return ret; /* No need of hypercall */ |
| 757 | + ret = vtl_set_vp_register(®); |
| 758 | + |
| 759 | + return ret; |
| 760 | +} |
| 761 | + |
| 762 | +static long |
| 763 | +mshv_vtl_ioctl(struct file *filp, unsigned int ioctl, unsigned long arg) |
| 764 | +{ |
| 765 | + long ret; |
| 766 | + struct mshv_vtl *vtl = filp->private_data; |
| 767 | + |
| 768 | + switch (ioctl) { |
| 769 | + case MSHV_SET_POLL_FILE: |
| 770 | + ret = mshv_vtl_ioctl_set_poll_file((struct mshv_vtl_set_poll_file __user *)arg); |
| 771 | + break; |
| 772 | + case MSHV_GET_VP_REGISTERS: |
| 773 | + ret = mshv_vtl_ioctl_get_regs((void __user *)arg); |
| 774 | + break; |
| 775 | + case MSHV_SET_VP_REGISTERS: |
| 776 | + ret = mshv_vtl_ioctl_set_regs((void __user *)arg); |
| 777 | + break; |
| 778 | + case MSHV_RETURN_TO_LOWER_VTL: |
| 779 | + ret = mshv_vtl_ioctl_return_to_lower_vtl(); |
| 780 | + break; |
| 781 | + case MSHV_ADD_VTL0_MEMORY: |
| 782 | + ret = mshv_vtl_ioctl_add_vtl0_mem(vtl, (void __user *)arg); |
| 783 | + break; |
| 784 | + default: |
| 785 | + dev_err(vtl->module_dev, "invalid vtl ioctl: %#x\n", ioctl); |
| 786 | + ret = -ENOTTY; |
| 787 | + } |
| 788 | + |
| 789 | + return ret; |
| 790 | +} |
| 791 | + |
| 792 | +static vm_fault_t mshv_vtl_fault(struct vm_fault *vmf) |
| 793 | +{ |
| 794 | + struct page *page; |
| 795 | + int cpu = vmf->pgoff & MSHV_PG_OFF_CPU_MASK; |
| 796 | + int real_off = vmf->pgoff >> MSHV_REAL_OFF_SHIFT; |
| 797 | + |
| 798 | + if (!cpu_online(cpu)) |
| 799 | + return VM_FAULT_SIGBUS; |
| 800 | + /* |
| 801 | + * CPU Hotplug is not supported in VTL2 in OpenHCL, where this kernel driver exists. |
| 802 | + * CPU is expected to remain online after above cpu_online() check. |
| 803 | + */ |
| 804 | + |
| 805 | + if (real_off == MSHV_RUN_PAGE_OFFSET) { |
| 806 | + page = virt_to_page(mshv_vtl_cpu_run(cpu)); |
| 807 | + } else if (real_off == MSHV_REG_PAGE_OFFSET) { |
| 808 | + if (!mshv_has_reg_page) |
| 809 | + return VM_FAULT_SIGBUS; |
| 810 | + page = mshv_vtl_cpu_reg_page(cpu); |
| 811 | + } else { |
| 812 | + return VM_FAULT_NOPAGE; |
| 813 | + } |
| 814 | + |
| 815 | + get_page(page); |
| 816 | + vmf->page = page; |
| 817 | + |
| 818 | + return 0; |
| 819 | +} |
| 820 | + |
| 821 | +static const struct vm_operations_struct mshv_vtl_vm_ops = { |
| 822 | + .fault = mshv_vtl_fault, |
| 823 | +}; |
| 824 | + |
| 825 | +static int mshv_vtl_mmap(struct file *filp, struct vm_area_struct *vma) |
| 826 | +{ |
| 827 | + vma->vm_ops = &mshv_vtl_vm_ops; |
| 828 | + |
| 829 | + return 0; |
| 830 | +} |
| 831 | + |
| 832 | +static int mshv_vtl_release(struct inode *inode, struct file *filp) |
| 833 | +{ |
| 834 | + struct mshv_vtl *vtl = filp->private_data; |
| 835 | + |
| 836 | + kfree(vtl); |
| 837 | + |
| 838 | + return 0; |
| 839 | +} |
| 840 | + |
| 841 | +static const struct file_operations mshv_vtl_fops = { |
| 842 | + .owner = THIS_MODULE, |
| 843 | + .unlocked_ioctl = mshv_vtl_ioctl, |
| 844 | + .release = mshv_vtl_release, |
| 845 | + .mmap = mshv_vtl_mmap, |
| 846 | +}; |
| 847 | + |
| 848 | +static void mshv_vtl_synic_mask_vmbus_sint(const u8 *mask) |
| 849 | +{ |
| 850 | + union hv_synic_sint sint; |
| 851 | + |
| 852 | + sint.as_uint64 = 0; |
| 853 | + sint.vector = HYPERVISOR_CALLBACK_VECTOR; |
| 854 | + sint.masked = (*mask != 0); |
| 855 | + sint.auto_eoi = hv_recommend_using_aeoi(); |
| 856 | + |
| 857 | + hv_set_msr(HV_MSR_SINT0 + VTL2_VMBUS_SINT_INDEX, |
| 858 | + sint.as_uint64); |
| 859 | + |
| 860 | + if (!sint.masked) |
| 861 | + pr_debug("%s: Unmasking VTL2 VMBUS SINT on VP %d\n", __func__, smp_processor_id()); |
| 862 | + else |
| 863 | + pr_debug("%s: Masking VTL2 VMBUS SINT on VP %d\n", __func__, smp_processor_id()); |
| 864 | +} |
| 865 | + |
| 866 | +static void mshv_vtl_read_remote(void *buffer) |
| 867 | +{ |
| 868 | + struct hv_per_cpu_context *mshv_cpu = this_cpu_ptr(hv_context.cpu_context); |
| 869 | + struct hv_message *msg = (struct hv_message *)mshv_cpu->hyp_synic_message_page + |
| 870 | + VTL2_VMBUS_SINT_INDEX; |
| 871 | + u32 message_type = READ_ONCE(msg->header.message_type); |
| 872 | + |
| 873 | + WRITE_ONCE(has_message, false); |
| 874 | + if (message_type == HVMSG_NONE) |
| 875 | + return; |
| 876 | + |
| 877 | + memcpy(buffer, msg, sizeof(*msg)); |
| 878 | + vmbus_signal_eom(msg, message_type); |
| 879 | +} |
| 880 | + |
| 881 | +static bool vtl_synic_mask_vmbus_sint_masked = true; |
| 882 | + |
| 883 | +static ssize_t mshv_vtl_sint_read(struct file *filp, char __user *arg, size_t size, loff_t *offset) |
| 884 | +{ |
| 885 | + struct hv_message msg = {}; |
| 886 | + int ret; |
| 887 | + |
| 888 | + if (size < sizeof(msg)) |
| 889 | + return -EINVAL; |
| 890 | + |
| 891 | + for (;;) { |
| 892 | + smp_call_function_single(VMBUS_CONNECT_CPU, mshv_vtl_read_remote, &msg, true); |
| 893 | + if (msg.header.message_type != HVMSG_NONE) |
| 894 | + break; |
| 895 | + |
| 896 | + if (READ_ONCE(vtl_synic_mask_vmbus_sint_masked)) |
| 897 | + return 0; /* EOF */ |
| 898 | + |
| 899 | + if (filp->f_flags & O_NONBLOCK) |
| 900 | + return -EAGAIN; |
| 901 | + |
| 902 | + ret = wait_event_interruptible(fd_wait_queue, |
| 903 | + READ_ONCE(has_message) || |
| 904 | + READ_ONCE(vtl_synic_mask_vmbus_sint_masked)); |
| 905 | + if (ret) |
| 906 | + return ret; |
| 907 | + } |
| 908 | + |
| 909 | + if (copy_to_user(arg, &msg, sizeof(msg))) |
| 910 | + return -EFAULT; |
| 911 | + |
| 912 | + return sizeof(msg); |
| 913 | +} |
| 914 | + |
| 915 | +static __poll_t mshv_vtl_sint_poll(struct file *filp, poll_table *wait) |
| 916 | +{ |
| 917 | + __poll_t mask = 0; |
| 918 | + |
| 919 | + poll_wait(filp, &fd_wait_queue, wait); |
| 920 | + if (READ_ONCE(has_message) || READ_ONCE(vtl_synic_mask_vmbus_sint_masked)) |
| 921 | + mask |= EPOLLIN | EPOLLRDNORM; |
| 922 | + |
| 923 | + return mask; |
| 924 | +} |
| 925 | + |
| 926 | +static void mshv_vtl_sint_on_msg_dpc(unsigned long data) |
| 927 | +{ |
| 928 | + WRITE_ONCE(has_message, true); |
| 929 | + wake_up_interruptible_poll(&fd_wait_queue, EPOLLIN); |
| 930 | +} |
| 931 | + |
| 932 | +static int mshv_vtl_sint_ioctl_post_msg(struct mshv_vtl_sint_post_msg __user *arg) |
| 933 | +{ |
| 934 | + struct mshv_vtl_sint_post_msg message; |
| 935 | + u8 payload[HV_MESSAGE_PAYLOAD_BYTE_COUNT]; |
| 936 | + |
| 937 | + if (copy_from_user(&message, arg, sizeof(message))) |
| 938 | + return -EFAULT; |
| 939 | + if (message.payload_size > HV_MESSAGE_PAYLOAD_BYTE_COUNT) |
| 940 | + return -EINVAL; |
| 941 | + if (copy_from_user(payload, (void __user *)message.payload_ptr, |
| 942 | + message.payload_size)) |
| 943 | + return -EFAULT; |
| 944 | + |
| 945 | + return hv_post_message((union hv_connection_id)message.connection_id, |
| 946 | + message.message_type, (void *)payload, |
| 947 | + message.payload_size); |
| 948 | +} |
| 949 | + |
| 950 | +static int mshv_vtl_sint_ioctl_signal_event(struct mshv_vtl_signal_event __user *arg) |
| 951 | +{ |
| 952 | + u64 input, status; |
| 953 | + struct mshv_vtl_signal_event signal_event; |
| 954 | + |
| 955 | + if (copy_from_user(&signal_event, arg, sizeof(signal_event))) |
| 956 | + return -EFAULT; |
| 957 | + |
| 958 | + input = signal_event.connection_id | ((u64)signal_event.flag << 32); |
| 959 | + |
| 960 | + status = hv_do_fast_hypercall8(HVCALL_SIGNAL_EVENT, input); |
| 961 | + |
| 962 | + return hv_result_to_errno(status); |
| 963 | +} |
| 964 | + |
| 965 | +static int mshv_vtl_sint_ioctl_set_eventfd(struct mshv_vtl_set_eventfd __user *arg) |
| 966 | +{ |
| 967 | + struct mshv_vtl_set_eventfd set_eventfd; |
| 968 | + struct eventfd_ctx *eventfd, *old_eventfd; |
| 969 | + |
| 970 | + if (copy_from_user(&set_eventfd, arg, sizeof(set_eventfd))) |
| 971 | + return -EFAULT; |
| 972 | + if (set_eventfd.flag >= HV_EVENT_FLAGS_COUNT) |
| 973 | + return -EINVAL; |
| 974 | + |
| 975 | + eventfd = NULL; |
| 976 | + if (set_eventfd.fd >= 0) { |
| 977 | + eventfd = eventfd_ctx_fdget(set_eventfd.fd); |
| 978 | + if (IS_ERR(eventfd)) |
| 979 | + return PTR_ERR(eventfd); |
| 980 | + } |
| 981 | + |
| 982 | + guard(mutex)(&flag_lock); |
| 983 | + old_eventfd = READ_ONCE(flag_eventfds[set_eventfd.flag]); |
| 984 | + WRITE_ONCE(flag_eventfds[set_eventfd.flag], eventfd); |
| 985 | + |
| 986 | + if (old_eventfd) { |
| 987 | + synchronize_rcu(); |
| 988 | + eventfd_ctx_put(old_eventfd); |
| 989 | + } |
| 990 | + |
| 991 | + return 0; |
| 992 | +} |
| 993 | + |
| 994 | +static int mshv_vtl_sint_ioctl_pause_msg_stream(struct mshv_sint_mask __user *arg) |
| 995 | +{ |
| 996 | + static DEFINE_MUTEX(vtl2_vmbus_sint_mask_mutex); |
| 997 | + struct mshv_sint_mask mask; |
| 998 | + |
| 999 | + if (copy_from_user(&mask, arg, sizeof(mask))) |
| 1000 | + return -EFAULT; |
| 1001 | + guard(mutex)(&vtl2_vmbus_sint_mask_mutex); |
| 1002 | + on_each_cpu((smp_call_func_t)mshv_vtl_synic_mask_vmbus_sint, &mask.mask, 1); |
| 1003 | + WRITE_ONCE(vtl_synic_mask_vmbus_sint_masked, mask.mask != 0); |
| 1004 | + if (mask.mask) |
| 1005 | + wake_up_interruptible_poll(&fd_wait_queue, EPOLLIN); |
| 1006 | + |
| 1007 | + return 0; |
| 1008 | +} |
| 1009 | + |
| 1010 | +static long mshv_vtl_sint_ioctl(struct file *f, unsigned int cmd, unsigned long arg) |
| 1011 | +{ |
| 1012 | + switch (cmd) { |
| 1013 | + case MSHV_SINT_POST_MESSAGE: |
| 1014 | + return mshv_vtl_sint_ioctl_post_msg((struct mshv_vtl_sint_post_msg __user *)arg); |
| 1015 | + case MSHV_SINT_SIGNAL_EVENT: |
| 1016 | + return mshv_vtl_sint_ioctl_signal_event((struct mshv_vtl_signal_event __user *)arg); |
| 1017 | + case MSHV_SINT_SET_EVENTFD: |
| 1018 | + return mshv_vtl_sint_ioctl_set_eventfd((struct mshv_vtl_set_eventfd __user *)arg); |
| 1019 | + case MSHV_SINT_PAUSE_MESSAGE_STREAM: |
| 1020 | + return mshv_vtl_sint_ioctl_pause_msg_stream((struct mshv_sint_mask __user *)arg); |
| 1021 | + default: |
| 1022 | + return -ENOIOCTLCMD; |
| 1023 | + } |
| 1024 | +} |
| 1025 | + |
| 1026 | +static const struct file_operations mshv_vtl_sint_ops = { |
| 1027 | + .owner = THIS_MODULE, |
| 1028 | + .read = mshv_vtl_sint_read, |
| 1029 | + .poll = mshv_vtl_sint_poll, |
| 1030 | + .unlocked_ioctl = mshv_vtl_sint_ioctl, |
| 1031 | +}; |
| 1032 | + |
| 1033 | +static struct miscdevice mshv_vtl_sint_dev = { |
| 1034 | + .name = "mshv_sint", |
| 1035 | + .fops = &mshv_vtl_sint_ops, |
| 1036 | + .mode = 0600, |
| 1037 | + .minor = MISC_DYNAMIC_MINOR, |
| 1038 | +}; |
| 1039 | + |
| 1040 | +static int mshv_vtl_hvcall_dev_open(struct inode *node, struct file *f) |
| 1041 | +{ |
| 1042 | + struct miscdevice *dev = f->private_data; |
| 1043 | + struct mshv_vtl_hvcall_fd *fd; |
| 1044 | + |
| 1045 | + if (!capable(CAP_SYS_ADMIN)) |
| 1046 | + return -EPERM; |
| 1047 | + |
| 1048 | + fd = vzalloc(sizeof(*fd)); |
| 1049 | + if (!fd) |
| 1050 | + return -ENOMEM; |
| 1051 | + fd->dev = dev; |
| 1052 | + f->private_data = fd; |
| 1053 | + mutex_init(&fd->init_mutex); |
| 1054 | + |
| 1055 | + return 0; |
| 1056 | +} |
| 1057 | + |
| 1058 | +static int mshv_vtl_hvcall_dev_release(struct inode *node, struct file *f) |
| 1059 | +{ |
| 1060 | + struct mshv_vtl_hvcall_fd *fd; |
| 1061 | + |
| 1062 | + fd = f->private_data; |
| 1063 | + if (fd) { |
| 1064 | + vfree(fd); |
| 1065 | + f->private_data = NULL; |
| 1066 | + } |
| 1067 | + |
| 1068 | + return 0; |
| 1069 | +} |
| 1070 | + |
| 1071 | +static int mshv_vtl_hvcall_do_setup(struct mshv_vtl_hvcall_fd *fd, |
| 1072 | + struct mshv_vtl_hvcall_setup __user *hvcall_setup_user) |
| 1073 | +{ |
| 1074 | + struct mshv_vtl_hvcall_setup hvcall_setup; |
| 1075 | + |
| 1076 | + guard(mutex)(&fd->init_mutex); |
| 1077 | + |
| 1078 | + if (fd->allow_map_initialized) { |
| 1079 | + dev_err(fd->dev->this_device, |
| 1080 | + "Hypercall allow map has already been set, pid %d\n", |
| 1081 | + current->pid); |
| 1082 | + return -EINVAL; |
| 1083 | + } |
| 1084 | + |
| 1085 | + if (copy_from_user(&hvcall_setup, hvcall_setup_user, |
| 1086 | + sizeof(struct mshv_vtl_hvcall_setup))) { |
| 1087 | + return -EFAULT; |
| 1088 | + } |
| 1089 | + if (hvcall_setup.bitmap_array_size > ARRAY_SIZE(fd->allow_bitmap)) |
| 1090 | + return -EINVAL; |
| 1091 | + |
| 1092 | + if (copy_from_user(&fd->allow_bitmap, |
| 1093 | + (void __user *)hvcall_setup.allow_bitmap_ptr, |
| 1094 | + hvcall_setup.bitmap_array_size)) { |
| 1095 | + return -EFAULT; |
| 1096 | + } |
| 1097 | + |
| 1098 | + dev_info(fd->dev->this_device, "Hypercall allow map has been set, pid %d\n", |
| 1099 | + current->pid); |
| 1100 | + fd->allow_map_initialized = true; |
| 1101 | + return 0; |
| 1102 | +} |
| 1103 | + |
| 1104 | +static bool mshv_vtl_hvcall_is_allowed(struct mshv_vtl_hvcall_fd *fd, u16 call_code) |
| 1105 | +{ |
| 1106 | + return test_bit(call_code, (unsigned long *)fd->allow_bitmap); |
| 1107 | +} |
| 1108 | + |
| 1109 | +static int mshv_vtl_hvcall_call(struct mshv_vtl_hvcall_fd *fd, |
| 1110 | + struct mshv_vtl_hvcall __user *hvcall_user) |
| 1111 | +{ |
| 1112 | + struct mshv_vtl_hvcall hvcall; |
| 1113 | + void *in, *out; |
| 1114 | + int ret; |
| 1115 | + |
| 1116 | + if (copy_from_user(&hvcall, hvcall_user, sizeof(struct mshv_vtl_hvcall))) |
| 1117 | + return -EFAULT; |
| 1118 | + if (hvcall.input_size > HV_HYP_PAGE_SIZE) |
| 1119 | + return -EINVAL; |
| 1120 | + if (hvcall.output_size > HV_HYP_PAGE_SIZE) |
| 1121 | + return -EINVAL; |
| 1122 | + |
| 1123 | + /* |
| 1124 | + * By default, all hypercalls are not allowed. |
| 1125 | + * The user mode code has to set up the allow bitmap once. |
| 1126 | + */ |
| 1127 | + |
| 1128 | + if (!mshv_vtl_hvcall_is_allowed(fd, hvcall.control & 0xFFFF)) { |
| 1129 | + dev_err(fd->dev->this_device, |
| 1130 | + "Hypercall with control data %#llx isn't allowed\n", |
| 1131 | + hvcall.control); |
| 1132 | + return -EPERM; |
| 1133 | + } |
| 1134 | + |
| 1135 | + /* |
| 1136 | + * This may create a problem for Confidential VM (CVM) usecase where we need to use |
| 1137 | + * Hyper-V driver allocated per-cpu input and output pages (hyperv_pcpu_input_arg and |
| 1138 | + * hyperv_pcpu_output_arg) for making a hypervisor call. |
| 1139 | + * |
| 1140 | + * TODO: Take care of this when CVM support is added. |
| 1141 | + */ |
| 1142 | + in = (void *)__get_free_page(GFP_KERNEL); |
| 1143 | + out = (void *)__get_free_page(GFP_KERNEL); |
| 1144 | + |
| 1145 | + if (copy_from_user(in, (void __user *)hvcall.input_ptr, hvcall.input_size)) { |
| 1146 | + ret = -EFAULT; |
| 1147 | + goto free_pages; |
| 1148 | + } |
| 1149 | + |
| 1150 | + hvcall.status = hv_do_hypercall(hvcall.control, in, out); |
| 1151 | + |
| 1152 | + if (copy_to_user((void __user *)hvcall.output_ptr, out, hvcall.output_size)) { |
| 1153 | + ret = -EFAULT; |
| 1154 | + goto free_pages; |
| 1155 | + } |
| 1156 | + ret = put_user(hvcall.status, &hvcall_user->status); |
| 1157 | +free_pages: |
| 1158 | + free_page((unsigned long)in); |
| 1159 | + free_page((unsigned long)out); |
| 1160 | + |
| 1161 | + return ret; |
| 1162 | +} |
| 1163 | + |
| 1164 | +static long mshv_vtl_hvcall_dev_ioctl(struct file *f, unsigned int cmd, unsigned long arg) |
| 1165 | +{ |
| 1166 | + struct mshv_vtl_hvcall_fd *fd = f->private_data; |
| 1167 | + |
| 1168 | + switch (cmd) { |
| 1169 | + case MSHV_HVCALL_SETUP: |
| 1170 | + return mshv_vtl_hvcall_do_setup(fd, (struct mshv_vtl_hvcall_setup __user *)arg); |
| 1171 | + case MSHV_HVCALL: |
| 1172 | + return mshv_vtl_hvcall_call(fd, (struct mshv_vtl_hvcall __user *)arg); |
| 1173 | + default: |
| 1174 | + break; |
| 1175 | + } |
| 1176 | + |
| 1177 | + return -ENOIOCTLCMD; |
| 1178 | +} |
| 1179 | + |
| 1180 | +static const struct file_operations mshv_vtl_hvcall_dev_file_ops = { |
| 1181 | + .owner = THIS_MODULE, |
| 1182 | + .open = mshv_vtl_hvcall_dev_open, |
| 1183 | + .release = mshv_vtl_hvcall_dev_release, |
| 1184 | + .unlocked_ioctl = mshv_vtl_hvcall_dev_ioctl, |
| 1185 | +}; |
| 1186 | + |
| 1187 | +static struct miscdevice mshv_vtl_hvcall_dev = { |
| 1188 | + .name = "mshv_hvcall", |
| 1189 | + .nodename = "mshv_hvcall", |
| 1190 | + .fops = &mshv_vtl_hvcall_dev_file_ops, |
| 1191 | + .mode = 0600, |
| 1192 | + .minor = MISC_DYNAMIC_MINOR, |
| 1193 | +}; |
| 1194 | + |
| 1195 | +static int mshv_vtl_low_open(struct inode *inodep, struct file *filp) |
| 1196 | +{ |
| 1197 | + pid_t pid = task_pid_vnr(current); |
| 1198 | + uid_t uid = current_uid().val; |
| 1199 | + int ret = 0; |
| 1200 | + |
| 1201 | + pr_debug("%s: Opening VTL low, task group %d, uid %d\n", __func__, pid, uid); |
| 1202 | + |
| 1203 | + if (capable(CAP_SYS_ADMIN)) { |
| 1204 | + filp->private_data = inodep; |
| 1205 | + } else { |
| 1206 | + pr_err("%s: VTL low open failed: CAP_SYS_ADMIN required. task group %d, uid %d", |
| 1207 | + __func__, pid, uid); |
| 1208 | + ret = -EPERM; |
| 1209 | + } |
| 1210 | + |
| 1211 | + return ret; |
| 1212 | +} |
| 1213 | + |
| 1214 | +static bool can_fault(struct vm_fault *vmf, unsigned long size, unsigned long *pfn) |
| 1215 | +{ |
| 1216 | + unsigned long mask = size - 1; |
| 1217 | + unsigned long start = vmf->address & ~mask; |
| 1218 | + unsigned long end = start + size; |
| 1219 | + bool is_valid; |
| 1220 | + |
| 1221 | + is_valid = (vmf->address & mask) == ((vmf->pgoff << PAGE_SHIFT) & mask) && |
| 1222 | + start >= vmf->vma->vm_start && |
| 1223 | + end <= vmf->vma->vm_end; |
| 1224 | + |
| 1225 | + if (is_valid) |
| 1226 | + *pfn = vmf->pgoff & ~(mask >> PAGE_SHIFT); |
| 1227 | + |
| 1228 | + return is_valid; |
| 1229 | +} |
| 1230 | + |
| 1231 | +static vm_fault_t mshv_vtl_low_huge_fault(struct vm_fault *vmf, unsigned int order) |
| 1232 | +{ |
| 1233 | + unsigned long pfn = vmf->pgoff; |
| 1234 | + vm_fault_t ret = VM_FAULT_FALLBACK; |
| 1235 | + |
| 1236 | + switch (order) { |
| 1237 | + case 0: |
| 1238 | + return vmf_insert_mixed(vmf->vma, vmf->address, pfn); |
| 1239 | + |
| 1240 | + case PMD_ORDER: |
| 1241 | + if (can_fault(vmf, PMD_SIZE, &pfn)) |
| 1242 | + ret = vmf_insert_pfn_pmd(vmf, pfn, vmf->flags & FAULT_FLAG_WRITE); |
| 1243 | + return ret; |
| 1244 | + |
| 1245 | + case PUD_ORDER: |
| 1246 | + if (can_fault(vmf, PUD_SIZE, &pfn)) |
| 1247 | + ret = vmf_insert_pfn_pud(vmf, pfn, vmf->flags & FAULT_FLAG_WRITE); |
| 1248 | + return ret; |
| 1249 | + |
| 1250 | + default: |
| 1251 | + return VM_FAULT_SIGBUS; |
| 1252 | + } |
| 1253 | +} |
| 1254 | + |
| 1255 | +static vm_fault_t mshv_vtl_low_fault(struct vm_fault *vmf) |
| 1256 | +{ |
| 1257 | + return mshv_vtl_low_huge_fault(vmf, 0); |
| 1258 | +} |
| 1259 | + |
| 1260 | +static const struct vm_operations_struct mshv_vtl_low_vm_ops = { |
| 1261 | + .fault = mshv_vtl_low_fault, |
| 1262 | + .huge_fault = mshv_vtl_low_huge_fault, |
| 1263 | +}; |
| 1264 | + |
| 1265 | +static int mshv_vtl_low_mmap(struct file *filp, struct vm_area_struct *vma) |
| 1266 | +{ |
| 1267 | + vma->vm_ops = &mshv_vtl_low_vm_ops; |
| 1268 | + vm_flags_set(vma, VM_HUGEPAGE | VM_MIXEDMAP); |
| 1269 | + |
| 1270 | + return 0; |
| 1271 | +} |
| 1272 | + |
| 1273 | +static const struct file_operations mshv_vtl_low_file_ops = { |
| 1274 | + .owner = THIS_MODULE, |
| 1275 | + .open = mshv_vtl_low_open, |
| 1276 | + .mmap = mshv_vtl_low_mmap, |
| 1277 | +}; |
| 1278 | + |
| 1279 | +static struct miscdevice mshv_vtl_low = { |
| 1280 | + .name = "mshv_vtl_low", |
| 1281 | + .nodename = "mshv_vtl_low", |
| 1282 | + .fops = &mshv_vtl_low_file_ops, |
| 1283 | + .mode = 0600, |
| 1284 | + .minor = MISC_DYNAMIC_MINOR, |
| 1285 | +}; |
| 1286 | + |
| 1287 | +static int __init mshv_vtl_init(void) |
| 1288 | +{ |
| 1289 | + int ret; |
| 1290 | + struct device *dev = mshv_dev.this_device; |
| 1291 | + |
| 1292 | + /* |
| 1293 | + * This creates /dev/mshv which provides functionality to create VTLs and partitions. |
| 1294 | + */ |
| 1295 | + ret = misc_register(&mshv_dev); |
| 1296 | + if (ret) { |
| 1297 | + dev_err(dev, "mshv device register failed: %d\n", ret); |
| 1298 | + goto free_dev; |
| 1299 | + } |
| 1300 | + |
| 1301 | + tasklet_init(&msg_dpc, mshv_vtl_sint_on_msg_dpc, 0); |
| 1302 | + init_waitqueue_head(&fd_wait_queue); |
| 1303 | + |
| 1304 | + if (mshv_vtl_get_vsm_regs()) { |
| 1305 | + dev_emerg(dev, "Unable to get VSM capabilities !!\n"); |
| 1306 | + ret = -ENODEV; |
| 1307 | + goto free_dev; |
| 1308 | + } |
| 1309 | + if (mshv_vtl_configure_vsm_partition(dev)) { |
| 1310 | + dev_emerg(dev, "VSM configuration failed !!\n"); |
| 1311 | + ret = -ENODEV; |
| 1312 | + goto free_dev; |
| 1313 | + } |
| 1314 | + |
| 1315 | + mshv_vtl_return_call_init(mshv_vsm_page_offsets.vtl_return_offset); |
| 1316 | + ret = hv_vtl_setup_synic(); |
| 1317 | + if (ret) |
| 1318 | + goto free_dev; |
| 1319 | + |
| 1320 | + /* |
| 1321 | + * mshv_sint device adds VMBus relay ioctl support. |
| 1322 | + * This provides a channel for VTL0 to communicate with VTL2. |
| 1323 | + */ |
| 1324 | + ret = misc_register(&mshv_vtl_sint_dev); |
| 1325 | + if (ret) |
| 1326 | + goto free_synic; |
| 1327 | + |
| 1328 | + /* |
| 1329 | + * mshv_hvcall device adds interface to enable userspace for direct hypercalls support. |
| 1330 | + */ |
| 1331 | + ret = misc_register(&mshv_vtl_hvcall_dev); |
| 1332 | + if (ret) |
| 1333 | + goto free_sint; |
| 1334 | + |
| 1335 | + /* |
| 1336 | + * mshv_vtl_low device is used to map VTL0 address space to a user-mode process in VTL2. |
| 1337 | + * It implements mmap() to allow a user-mode process in VTL2 to map to the address of VTL0. |
| 1338 | + */ |
| 1339 | + ret = misc_register(&mshv_vtl_low); |
| 1340 | + if (ret) |
| 1341 | + goto free_hvcall; |
| 1342 | + |
| 1343 | + /* |
| 1344 | + * "mshv vtl mem dev" device is later used to setup VTL0 memory. |
| 1345 | + */ |
| 1346 | + mem_dev = kzalloc(sizeof(*mem_dev), GFP_KERNEL); |
| 1347 | + if (!mem_dev) { |
| 1348 | + ret = -ENOMEM; |
| 1349 | + goto free_low; |
| 1350 | + } |
| 1351 | + |
| 1352 | + mutex_init(&mshv_vtl_poll_file_lock); |
| 1353 | + |
| 1354 | + device_initialize(mem_dev); |
| 1355 | + dev_set_name(mem_dev, "mshv vtl mem dev"); |
| 1356 | + ret = device_add(mem_dev); |
| 1357 | + if (ret) { |
| 1358 | + dev_err(dev, "mshv vtl mem dev add: %d\n", ret); |
| 1359 | + goto free_mem; |
| 1360 | + } |
| 1361 | + |
| 1362 | + return 0; |
| 1363 | + |
| 1364 | +free_mem: |
| 1365 | + kfree(mem_dev); |
| 1366 | +free_low: |
| 1367 | + misc_deregister(&mshv_vtl_low); |
| 1368 | +free_hvcall: |
| 1369 | + misc_deregister(&mshv_vtl_hvcall_dev); |
| 1370 | +free_sint: |
| 1371 | + misc_deregister(&mshv_vtl_sint_dev); |
| 1372 | +free_synic: |
| 1373 | + hv_vtl_remove_synic(); |
| 1374 | +free_dev: |
| 1375 | + misc_deregister(&mshv_dev); |
| 1376 | + |
| 1377 | + return ret; |
| 1378 | +} |
| 1379 | + |
| 1380 | +static void __exit mshv_vtl_exit(void) |
| 1381 | +{ |
| 1382 | + device_del(mem_dev); |
| 1383 | + kfree(mem_dev); |
| 1384 | + misc_deregister(&mshv_vtl_low); |
| 1385 | + misc_deregister(&mshv_vtl_hvcall_dev); |
| 1386 | + misc_deregister(&mshv_vtl_sint_dev); |
| 1387 | + hv_vtl_remove_synic(); |
| 1388 | + misc_deregister(&mshv_dev); |
| 1389 | +} |
| 1390 | + |
| 1391 | +module_init(mshv_vtl_init); |
| 1392 | +module_exit(mshv_vtl_exit); |