History

KVM: Introduce per-page memory attributes

In confidential computing usages, whether a page is private or shared is necessary information for KVM to perform operations like page fault handling, page zapping etc. There are other potential use cases for per-page memory attributes, e.g. to make memory read-only (or no-exec, or exec-only, etc.) without having to modify memslots. Introduce the KVM_SET_MEMORY_ATTRIBUTES ioctl, advertised by KVM_CAP_MEMORY_ATTRIBUTES, to allow userspace to set the per-page memory attributes to a guest memory range. Use an xarray to store the per-page attributes internally, with a naive, not fully optimized implementation, i.e. prioritize correctness over performance for the initial implementation. Use bit 3 for the PRIVATE attribute so that KVM can use bits 0-2 for RWX attributes/protections in the future, e.g. to give userspace fine-grained control over read, write, and execute protections for guest memory. Provide arch hooks for handling attribute changes before and after common code sets the new attributes, e.g. x86 will use the "pre" hook to zap all relevant mappings, and the "post" hook to track whether or not hugepages can be used to map the range. To simplify the implementation wrap the entire sequence with kvm_mmu_invalidate_{begin,end}() even though the operation isn't strictly guaranteed to be an invalidation. For the initial use case, x86 *will* always invalidate memory, and preventing arch code from creating new mappings while the attributes are in flux makes it much easier to reason about the correctness of consuming attributes. It's possible that future usages may not require an invalidation, e.g. if KVM ends up supporting RWX protections and userspace grants _more_ protections, but again opt for simplicity and punt optimizations to if/when they are needed.

Chao Peng committed 3 years ago

Commit 5a475551 parent 193bbfa5 files · +288 −0Browse files

Unified · Split

5 files
  1. Documentation/virt/kvm/api.rst +36 −0
  2. include/linux/kvm_host.h +19 −0
  3. include/uapi/linux/kvm.h +13 −0
  4. virt/kvm/Kconfig +4 −0
  5. virt/kvm/kvm_main.c +216 −0
Documentation/virt/kvm/api.rst+36−0
@@ -6214,6 +6214,42 @@ superset of the features supported by the system. · 6213 unmodified lines
6214 6214
6215See KVM_SET_USER_MEMORY_REGION.6215See KVM_SET_USER_MEMORY_REGION.
6216 6216
62174.141 KVM_SET_MEMORY_ATTRIBUTES
6218-------------------------------
6219
6220:Capability: KVM_CAP_MEMORY_ATTRIBUTES
6221:Architectures: x86
6222:Type: vm ioctl
6223:Parameters: struct kvm_memory_attributes (in)
6224:Returns: 0 on success, <0 on error
6225
6226KVM_SET_MEMORY_ATTRIBUTES allows userspace to set memory attributes for a range
6227of guest physical memory.
6228
6229::
6230
6231 struct kvm_memory_attributes {
6232 __u64 address;
6233 __u64 size;
6234 __u64 attributes;
6235 __u64 flags;
6236 };
6237
6238 #define KVM_MEMORY_ATTRIBUTE_PRIVATE (1ULL << 3)
6239
6240The address and size must be page aligned. The supported attributes can be
6241retrieved via ioctl(KVM_CHECK_EXTENSION) on KVM_CAP_MEMORY_ATTRIBUTES. If
6242executed on a VM, KVM_CAP_MEMORY_ATTRIBUTES precisely returns the attributes
6243supported by that VM. If executed at system scope, KVM_CAP_MEMORY_ATTRIBUTES
6244returns all attributes supported by KVM. The only attribute defined at this
6245time is KVM_MEMORY_ATTRIBUTE_PRIVATE, which marks the associated gfn as being
6246guest private memory.
6247
6248Note, there is no "get" API. Userspace is responsible for explicitly tracking
6249the state of a gfn/page as needed.
6250
6251The "flags" field is reserved for future extensions and must be '0'.
6252
62175. The kvm_run structure62535. The kvm_run structure
6218========================6254========================
6219 6255
include/linux/kvm_host.h+19−0
@@ -256,6 +256,7 @@ int kvm_async_pf_wakeup_all(struct kvm_vcpu *vcpu); · 255 unmodified lines
256#ifdef CONFIG_KVM_GENERIC_MMU_NOTIFIER256#ifdef CONFIG_KVM_GENERIC_MMU_NOTIFIER
257union kvm_mmu_notifier_arg {257union kvm_mmu_notifier_arg {
258 pte_t pte;258 pte_t pte;
259 unsigned long attributes;
259};260};
260 261
261struct kvm_gfn_range {262struct kvm_gfn_range {
@@ -806,6 +807,10 @@ struct kvm { · 544 unmodified lines
806 807
807#ifdef CONFIG_HAVE_KVM_PM_NOTIFIER808#ifdef CONFIG_HAVE_KVM_PM_NOTIFIER
808 struct notifier_block pm_notifier;809 struct notifier_block pm_notifier;
810#endif
811#ifdef CONFIG_KVM_GENERIC_MEMORY_ATTRIBUTES
812 /* Protected by slots_locks (for writes) and RCU (for reads) */
813 struct xarray mem_attr_array;
809#endif814#endif
810 char stats_id[KVM_STATS_NAME_SIZE];815 char stats_id[KVM_STATS_NAME_SIZE];
811};816};
@@ -2338,4 +2343,18 @@ static inline void kvm_prepare_memory_fault_exit(struct kvm_vcpu *vcpu, · 1526 unmodified lines
2338 vcpu->run->memory_fault.flags = 0;2343 vcpu->run->memory_fault.flags = 0;
2339}2344}
2340 2345
2346#ifdef CONFIG_KVM_GENERIC_MEMORY_ATTRIBUTES
2347static inline unsigned long kvm_get_memory_attributes(struct kvm *kvm, gfn_t gfn)
2348{
2349 return xa_to_value(xa_load(&kvm->mem_attr_array, gfn));
2350}
2351
2352bool kvm_range_has_memory_attributes(struct kvm *kvm, gfn_t start, gfn_t end,
2353 unsigned long attrs);
2354bool kvm_arch_pre_set_memory_attributes(struct kvm *kvm,
2355 struct kvm_gfn_range *range);
2356bool kvm_arch_post_set_memory_attributes(struct kvm *kvm,
2357 struct kvm_gfn_range *range);
2358#endif /* CONFIG_KVM_GENERIC_MEMORY_ATTRIBUTES */
2359
2341#endif2360#endif
include/uapi/linux/kvm.h+13−0
@@ -1220,6 +1220,7 @@ struct kvm_ppc_resize_hpt { · 1219 unmodified lines
1220#define KVM_CAP_ARM_SUPPORTED_REG_MASK_RANGES 2301220#define KVM_CAP_ARM_SUPPORTED_REG_MASK_RANGES 230
1221#define KVM_CAP_USER_MEMORY2 2311221#define KVM_CAP_USER_MEMORY2 231
1222#define KVM_CAP_MEMORY_FAULT_INFO 2321222#define KVM_CAP_MEMORY_FAULT_INFO 232
1223#define KVM_CAP_MEMORY_ATTRIBUTES 233
1223 1224
1224#ifdef KVM_CAP_IRQ_ROUTING1225#ifdef KVM_CAP_IRQ_ROUTING
1225 1226
@@ -2288,4 +2289,16 @@ struct kvm_s390_zpci_op { · 1062 unmodified lines
2288/* flags for kvm_s390_zpci_op->u.reg_aen.flags */2289/* flags for kvm_s390_zpci_op->u.reg_aen.flags */
2289#define KVM_S390_ZPCIOP_REGAEN_HOST (1 << 0)2290#define KVM_S390_ZPCIOP_REGAEN_HOST (1 << 0)
2290 2291
2292/* Available with KVM_CAP_MEMORY_ATTRIBUTES */
2293#define KVM_SET_MEMORY_ATTRIBUTES _IOW(KVMIO, 0xd2, struct kvm_memory_attributes)
2294
2295struct kvm_memory_attributes {
2296 __u64 address;
2297 __u64 size;
2298 __u64 attributes;
2299 __u64 flags;
2300};
2301
2302#define KVM_MEMORY_ATTRIBUTE_PRIVATE (1ULL << 3)
2303
2291#endif /* __LINUX_KVM_H */2304#endif /* __LINUX_KVM_H */
virt/kvm/Kconfig+4−0
@@ -96,3 +96,7 @@ config KVM_GENERIC_HARDWARE_ENABLING · 95 unmodified lines
96config KVM_GENERIC_MMU_NOTIFIER96config KVM_GENERIC_MMU_NOTIFIER
97 select MMU_NOTIFIER97 select MMU_NOTIFIER
98 bool98 bool
99
100config KVM_GENERIC_MEMORY_ATTRIBUTES
101 select KVM_GENERIC_MMU_NOTIFIER
102 bool
virt/kvm/kvm_main.c+216−0
@@ -1211,6 +1211,9 @@ static struct kvm *kvm_create_vm(unsigned long type, const char *fdname) · 1210 unmodified lines
1211 spin_lock_init(&kvm->mn_invalidate_lock);1211 spin_lock_init(&kvm->mn_invalidate_lock);
1212 rcuwait_init(&kvm->mn_memslots_update_rcuwait);1212 rcuwait_init(&kvm->mn_memslots_update_rcuwait);
1213 xa_init(&kvm->vcpu_array);1213 xa_init(&kvm->vcpu_array);
1214#ifdef CONFIG_KVM_GENERIC_MEMORY_ATTRIBUTES
1215 xa_init(&kvm->mem_attr_array);
1216#endif
1214 1217
1215 INIT_LIST_HEAD(&kvm->gpc_list);1218 INIT_LIST_HEAD(&kvm->gpc_list);
1216 spin_lock_init(&kvm->gpc_lock);1219 spin_lock_init(&kvm->gpc_lock);
@@ -1391,6 +1394,9 @@ static void kvm_destroy_vm(struct kvm *kvm) · 174 unmodified lines
1391 }1394 }
1392 cleanup_srcu_struct(&kvm->irq_srcu);1395 cleanup_srcu_struct(&kvm->irq_srcu);
1393 cleanup_srcu_struct(&kvm->srcu);1396 cleanup_srcu_struct(&kvm->srcu);
1397#ifdef CONFIG_KVM_GENERIC_MEMORY_ATTRIBUTES
1398 xa_destroy(&kvm->mem_attr_array);
1399#endif
1394 kvm_arch_free_vm(kvm);1400 kvm_arch_free_vm(kvm);
1395 preempt_notifier_dec();1401 preempt_notifier_dec();
1396 hardware_disable_all();1402 hardware_disable_all();
@@ -2397,6 +2403,200 @@ static int kvm_vm_ioctl_clear_dirty_log(struct kvm *kvm, · 1000 unmodified lines
2397}2403}
2398#endif /* CONFIG_KVM_GENERIC_DIRTYLOG_READ_PROTECT */2404#endif /* CONFIG_KVM_GENERIC_DIRTYLOG_READ_PROTECT */
2399 2405
2406#ifdef CONFIG_KVM_GENERIC_MEMORY_ATTRIBUTES
2407/*
2408 * Returns true if _all_ gfns in the range [@start, @end) have attributes
2409 * matching @attrs.
2410 */
2411bool kvm_range_has_memory_attributes(struct kvm *kvm, gfn_t start, gfn_t end,
2412 unsigned long attrs)
2413{
2414 XA_STATE(xas, &kvm->mem_attr_array, start);
2415 unsigned long index;
2416 bool has_attrs;
2417 void *entry;
2418
2419 rcu_read_lock();
2420
2421 if (!attrs) {
2422 has_attrs = !xas_find(&xas, end - 1);
2423 goto out;
2424 }
2425
2426 has_attrs = true;
2427 for (index = start; index < end; index++) {
2428 do {
2429 entry = xas_next(&xas);
2430 } while (xas_retry(&xas, entry));
2431
2432 if (xas.xa_index != index || xa_to_value(entry) != attrs) {
2433 has_attrs = false;
2434 break;
2435 }
2436 }
2437
2438out:
2439 rcu_read_unlock();
2440 return has_attrs;
2441}
2442
2443static u64 kvm_supported_mem_attributes(struct kvm *kvm)
2444{
2445 if (!kvm)
2446 return KVM_MEMORY_ATTRIBUTE_PRIVATE;
2447
2448 return 0;
2449}
2450
2451static __always_inline void kvm_handle_gfn_range(struct kvm *kvm,
2452 struct kvm_mmu_notifier_range *range)
2453{
2454 struct kvm_gfn_range gfn_range;
2455 struct kvm_memory_slot *slot;
2456 struct kvm_memslots *slots;
2457 struct kvm_memslot_iter iter;
2458 bool found_memslot = false;
2459 bool ret = false;
2460 int i;
2461
2462 gfn_range.arg = range->arg;
2463 gfn_range.may_block = range->may_block;
2464
2465 for (i = 0; i < KVM_ADDRESS_SPACE_NUM; i++) {
2466 slots = __kvm_memslots(kvm, i);
2467
2468 kvm_for_each_memslot_in_gfn_range(&iter, slots, range->start, range->end) {
2469 slot = iter.slot;
2470 gfn_range.slot = slot;
2471
2472 gfn_range.start = max(range->start, slot->base_gfn);
2473 gfn_range.end = min(range->end, slot->base_gfn + slot->npages);
2474 if (gfn_range.start >= gfn_range.end)
2475 continue;
2476
2477 if (!found_memslot) {
2478 found_memslot = true;
2479 KVM_MMU_LOCK(kvm);
2480 if (!IS_KVM_NULL_FN(range->on_lock))
2481 range->on_lock(kvm);
2482 }
2483
2484 ret |= range->handler(kvm, &gfn_range);
2485 }
2486 }
2487
2488 if (range->flush_on_ret && ret)
2489 kvm_flush_remote_tlbs(kvm);
2490
2491 if (found_memslot)
2492 KVM_MMU_UNLOCK(kvm);
2493}
2494
2495static bool kvm_pre_set_memory_attributes(struct kvm *kvm,
2496 struct kvm_gfn_range *range)
2497{
2498 /*
2499 * Unconditionally add the range to the invalidation set, regardless of
2500 * whether or not the arch callback actually needs to zap SPTEs. E.g.
2501 * if KVM supports RWX attributes in the future and the attributes are
2502 * going from R=>RW, zapping isn't strictly necessary. Unconditionally
2503 * adding the range allows KVM to require that MMU invalidations add at
2504 * least one range between begin() and end(), e.g. allows KVM to detect
2505 * bugs where the add() is missed. Relaxing the rule *might* be safe,
2506 * but it's not obvious that allowing new mappings while the attributes
2507 * are in flux is desirable or worth the complexity.
2508 */
2509 kvm_mmu_invalidate_range_add(kvm, range->start, range->end);
2510
2511 return kvm_arch_pre_set_memory_attributes(kvm, range);
2512}
2513
2514/* Set @attributes for the gfn range [@start, @end). */
2515static int kvm_vm_set_mem_attributes(struct kvm *kvm, gfn_t start, gfn_t end,
2516 unsigned long attributes)
2517{
2518 struct kvm_mmu_notifier_range pre_set_range = {
2519 .start = start,
2520 .end = end,
2521 .handler = kvm_pre_set_memory_attributes,
2522 .on_lock = kvm_mmu_invalidate_begin,
2523 .flush_on_ret = true,
2524 .may_block = true,
2525 };
2526 struct kvm_mmu_notifier_range post_set_range = {
2527 .start = start,
2528 .end = end,
2529 .arg.attributes = attributes,
2530 .handler = kvm_arch_post_set_memory_attributes,
2531 .on_lock = kvm_mmu_invalidate_end,
2532 .may_block = true,
2533 };
2534 unsigned long i;
2535 void *entry;
2536 int r = 0;
2537
2538 entry = attributes ? xa_mk_value(attributes) : NULL;
2539
2540 mutex_lock(&kvm->slots_lock);
2541
2542 /* Nothing to do if the entire range as the desired attributes. */
2543 if (kvm_range_has_memory_attributes(kvm, start, end, attributes))
2544 goto out_unlock;
2545
2546 /*
2547 * Reserve memory ahead of time to avoid having to deal with failures
2548 * partway through setting the new attributes.
2549 */
2550 for (i = start; i < end; i++) {
2551 r = xa_reserve(&kvm->mem_attr_array, i, GFP_KERNEL_ACCOUNT);
2552 if (r)
2553 goto out_unlock;
2554 }
2555
2556 kvm_handle_gfn_range(kvm, &pre_set_range);
2557
2558 for (i = start; i < end; i++) {
2559 r = xa_err(xa_store(&kvm->mem_attr_array, i, entry,
2560 GFP_KERNEL_ACCOUNT));
2561 KVM_BUG_ON(r, kvm);
2562 }
2563
2564 kvm_handle_gfn_range(kvm, &post_set_range);
2565
2566out_unlock:
2567 mutex_unlock(&kvm->slots_lock);
2568
2569 return r;
2570}
2571static int kvm_vm_ioctl_set_mem_attributes(struct kvm *kvm,
2572 struct kvm_memory_attributes *attrs)
2573{
2574 gfn_t start, end;
2575
2576 /* flags is currently not used. */
2577 if (attrs->flags)
2578 return -EINVAL;
2579 if (attrs->attributes & ~kvm_supported_mem_attributes(kvm))
2580 return -EINVAL;
2581 if (attrs->size == 0 || attrs->address + attrs->size < attrs->address)
2582 return -EINVAL;
2583 if (!PAGE_ALIGNED(attrs->address) || !PAGE_ALIGNED(attrs->size))
2584 return -EINVAL;
2585
2586 start = attrs->address >> PAGE_SHIFT;
2587 end = (attrs->address + attrs->size) >> PAGE_SHIFT;
2588
2589 /*
2590 * xarray tracks data using "unsigned long", and as a result so does
2591 * KVM. For simplicity, supports generic attributes only on 64-bit
2592 * architectures.
2593 */
2594 BUILD_BUG_ON(sizeof(attrs->attributes) != sizeof(unsigned long));
2595
2596 return kvm_vm_set_mem_attributes(kvm, start, end, attrs->attributes);
2597}
2598#endif /* CONFIG_KVM_GENERIC_MEMORY_ATTRIBUTES */
2599
2400struct kvm_memory_slot *gfn_to_memslot(struct kvm *kvm, gfn_t gfn)2600struct kvm_memory_slot *gfn_to_memslot(struct kvm *kvm, gfn_t gfn)
2401{2601{
2402 return __gfn_to_memslot(kvm_memslots(kvm), gfn);2602 return __gfn_to_memslot(kvm_memslots(kvm), gfn);
@@ -4641,6 +4841,10 @@ static int kvm_vm_ioctl_check_extension_generic(struct kvm *kvm, long arg) · 2238 unmodified lines
4641 case KVM_CAP_BINARY_STATS_FD:4841 case KVM_CAP_BINARY_STATS_FD:
4642 case KVM_CAP_SYSTEM_EVENT_DATA:4842 case KVM_CAP_SYSTEM_EVENT_DATA:
4643 return 1;4843 return 1;
4844#ifdef CONFIG_KVM_GENERIC_MEMORY_ATTRIBUTES
4845 case KVM_CAP_MEMORY_ATTRIBUTES:
4846 return kvm_supported_mem_attributes(kvm);
4847#endif
4644 default:4848 default:
4645 break;4849 break;
4646 }4850 }
@@ -5034,6 +5238,18 @@ static long kvm_vm_ioctl(struct file *filp, · 387 unmodified lines
5034 break;5238 break;
5035 }5239 }
5036#endif /* CONFIG_HAVE_KVM_IRQ_ROUTING */5240#endif /* CONFIG_HAVE_KVM_IRQ_ROUTING */
5241#ifdef CONFIG_KVM_GENERIC_MEMORY_ATTRIBUTES
5242 case KVM_SET_MEMORY_ATTRIBUTES: {
5243 struct kvm_memory_attributes attrs;
5244
5245 r = -EFAULT;
5246 if (copy_from_user(&attrs, argp, sizeof(attrs)))
5247 goto out;
5248
5249 r = kvm_vm_ioctl_set_mem_attributes(kvm, &attrs);
5250 break;
5251 }
5252#endif /* CONFIG_KVM_GENERIC_MEMORY_ATTRIBUTES */
5037 case KVM_CREATE_DEVICE: {5253 case KVM_CREATE_DEVICE: {
5038 struct kvm_create_device cd;5254 struct kvm_create_device cd;
5039 5255

Keyboard shortcuts

General

Show keyboard shortcuts
?
Focus the current search
/

Issues and pull requests

Open filters
f

Issues, pull requests and Inbox

Select next item
j↓
Select previous item
k↑
Open selected item
Enter
Dismiss filters or clear selection
Esc

Repository code

Go to file
t