With KVM_CAP_XSAVE, userspace uses a hardcoded 4KB buffer to get/set xstate data from/to KVM. This doesn't work when dynamic xfeatures (e.g. AMX) are exposed to the guest as they require a larger buffer size. Introduce a new capability (KVM_CAP_XSAVE2). Userspace VMM gets the required xstate buffer size via KVM_CHECK_EXTENSION(KVM_CAP_XSAVE2). KVM_SET_XSAVE is extended to work with both legacy and new capabilities by doing properly-sized memdup_user() based on the guest fpu container. KVM_GET_XSAVE is kept for backward-compatible reason. Instead, KVM_GET_XSAVE2 is introduced under KVM_CAP_XSAVE2 as the preferred interface for getting xstate buffer (4KB or larger size) from KVM (Link: https://lkml.org/lkml/2021/12/15/510) Also, update the api doc with the new KVM_GET_XSAVE2 ioctl.
Unified · Split
| @@ -1569,6 +1569,7 @@ otherwise it will return EBUSY error. · 1568 unmodified lines | |||
| 1569 | 1569 | ||
| 1570 | struct kvm_xsave { | 1570 | struct kvm_xsave { |
| 1571 | __u32 region[1024]; | 1571 | __u32 region[1024]; |
| 1572 | __u32 extra[0]; | ||
| 1572 | }; | 1573 | }; |
| 1573 | 1574 | ||
| 1574 | This ioctl would copy current vcpu's xsave struct to the userspace. | 1575 | This ioctl would copy current vcpu's xsave struct to the userspace. |
| @@ -1577,7 +1578,7 @@ This ioctl would copy current vcpu's xsave struct to the userspace. · 2 unmodified lines | |||
| 1577 | 4.43 KVM_SET_XSAVE | 1578 | 4.43 KVM_SET_XSAVE |
| 1578 | ------------------ | 1579 | ------------------ |
| 1579 | 1580 | ||
| 1580 | :Capability: KVM_CAP_XSAVE | 1581 | :Capability: KVM_CAP_XSAVE and KVM_CAP_XSAVE2 |
| 1581 | :Architectures: x86 | 1582 | :Architectures: x86 |
| 1582 | :Type: vcpu ioctl | 1583 | :Type: vcpu ioctl |
| 1583 | :Parameters: struct kvm_xsave (in) | 1584 | :Parameters: struct kvm_xsave (in) |
| @@ -1588,9 +1589,18 @@ This ioctl would copy current vcpu's xsave struct to the userspace. · 4 unmodified lines | |||
| 1588 | 1589 | ||
| 1589 | struct kvm_xsave { | 1590 | struct kvm_xsave { |
| 1590 | __u32 region[1024]; | 1591 | __u32 region[1024]; |
| 1592 | __u32 extra[0]; | ||
| 1591 | }; | 1593 | }; |
| 1592 | 1594 | ||
| 1593 | This ioctl would copy userspace's xsave struct to the kernel. | 1595 | This ioctl would copy userspace's xsave struct to the kernel. It copies |
| 1596 | as many bytes as are returned by KVM_CHECK_EXTENSION(KVM_CAP_XSAVE2), | ||
| 1597 | when invoked on the vm file descriptor. The size value returned by | ||
| 1598 | KVM_CHECK_EXTENSION(KVM_CAP_XSAVE2) will always be at least 4096. | ||
| 1599 | Currently, it is only greater than 4096 if a dynamic feature has been | ||
| 1600 | enabled with ``arch_prctl()``, but this may change in the future. | ||
| 1601 | |||
| 1602 | The offsets of the state save areas in struct kvm_xsave follow the | ||
| 1603 | contents of CPUID leaf 0xD on the host. | ||
| 1594 | 1604 | ||
| 1595 | 1605 | ||
| 1596 | 4.44 KVM_GET_XCRS | 1606 | 4.44 KVM_GET_XCRS |
| @@ -5535,6 +5545,34 @@ the trailing ``'\0'``, is indicated by ``name_size`` in the header. · 3938 unmodified lines | |||
| 5535 | The Stats Data block contains an array of 64-bit values in the same order | 5545 | The Stats Data block contains an array of 64-bit values in the same order |
| 5536 | as the descriptors in Descriptors block. | 5546 | as the descriptors in Descriptors block. |
| 5537 | 5547 | ||
| 5548 | 4.42 KVM_GET_XSAVE2 | ||
| 5549 | ------------------ | ||
| 5550 | |||
| 5551 | :Capability: KVM_CAP_XSAVE2 | ||
| 5552 | :Architectures: x86 | ||
| 5553 | :Type: vcpu ioctl | ||
| 5554 | :Parameters: struct kvm_xsave (out) | ||
| 5555 | :Returns: 0 on success, -1 on error | ||
| 5556 | |||
| 5557 | |||
| 5558 | :: | ||
| 5559 | |||
| 5560 | struct kvm_xsave { | ||
| 5561 | __u32 region[1024]; | ||
| 5562 | __u32 extra[0]; | ||
| 5563 | }; | ||
| 5564 | |||
| 5565 | This ioctl would copy current vcpu's xsave struct to the userspace. It | ||
| 5566 | copies as many bytes as are returned by KVM_CHECK_EXTENSION(KVM_CAP_XSAVE2) | ||
| 5567 | when invoked on the vm file descriptor. The size value returned by | ||
| 5568 | KVM_CHECK_EXTENSION(KVM_CAP_XSAVE2) will always be at least 4096. | ||
| 5569 | Currently, it is only greater than 4096 if a dynamic feature has been | ||
| 5570 | enabled with ``arch_prctl()``, but this may change in the future. | ||
| 5571 | |||
| 5572 | The offsets of the state save areas in struct kvm_xsave follow the contents | ||
| 5573 | of CPUID leaf 0xD on the host. | ||
| 5574 | |||
| 5575 | |||
| 5538 | 5. The kvm_run structure | 5576 | 5. The kvm_run structure |
| 5539 | ======================== | 5577 | ======================== |
| 5540 | 5578 | ||
| @@ -373,9 +373,23 @@ struct kvm_debugregs { · 372 unmodified lines | |||
| 373 | __u64 reserved[9]; | 373 | __u64 reserved[9]; |
| 374 | }; | 374 | }; |
| 375 | 375 | ||
| 376 | /* for KVM_CAP_XSAVE */ | 376 | /* for KVM_CAP_XSAVE and KVM_CAP_XSAVE2 */ |
| 377 | struct kvm_xsave { | 377 | struct kvm_xsave { |
| 378 | /* | ||
| 379 | * KVM_GET_XSAVE2 and KVM_SET_XSAVE write and read as many bytes | ||
| 380 | * as are returned by KVM_CHECK_EXTENSION(KVM_CAP_XSAVE2) | ||
| 381 | * respectively, when invoked on the vm file descriptor. | ||
| 382 | * | ||
| 383 | * The size value returned by KVM_CHECK_EXTENSION(KVM_CAP_XSAVE2) | ||
| 384 | * will always be at least 4096. Currently, it is only greater | ||
| 385 | * than 4096 if a dynamic feature has been enabled with | ||
| 386 | * ``arch_prctl()``, but this may change in the future. | ||
| 387 | * | ||
| 388 | * The offsets of the state save areas in struct kvm_xsave follow | ||
| 389 | * the contents of CPUID leaf 0xD on the host. | ||
| 390 | */ | ||
| 378 | __u32 region[1024]; | 391 | __u32 region[1024]; |
| 392 | __u32 extra[0]; | ||
| 379 | }; | 393 | }; |
| 380 | 394 | ||
| 381 | #define KVM_MAX_XCRS 16 | 395 | #define KVM_MAX_XCRS 16 |
| @@ -32,7 +32,7 @@ · 31 unmodified lines | |||
| 32 | u32 kvm_cpu_caps[NR_KVM_CPU_CAPS] __read_mostly; | 32 | u32 kvm_cpu_caps[NR_KVM_CPU_CAPS] __read_mostly; |
| 33 | EXPORT_SYMBOL_GPL(kvm_cpu_caps); | 33 | EXPORT_SYMBOL_GPL(kvm_cpu_caps); |
| 34 | 34 | ||
| 35 | static u32 xstate_required_size(u64 xstate_bv, bool compacted) | 35 | u32 xstate_required_size(u64 xstate_bv, bool compacted) |
| 36 | { | 36 | { |
| 37 | int feature_bit = 0; | 37 | int feature_bit = 0; |
| 38 | u32 ret = XSAVE_HDR_SIZE + XSAVE_HDR_OFFSET; | 38 | u32 ret = XSAVE_HDR_SIZE + XSAVE_HDR_OFFSET; |
| @@ -30,6 +30,8 @@ int kvm_vcpu_ioctl_get_cpuid2(struct kvm_vcpu *vcpu, · 29 unmodified lines | |||
| 30 | bool kvm_cpuid(struct kvm_vcpu *vcpu, u32 *eax, u32 *ebx, | 30 | bool kvm_cpuid(struct kvm_vcpu *vcpu, u32 *eax, u32 *ebx, |
| 31 | u32 *ecx, u32 *edx, bool exact_only); | 31 | u32 *ecx, u32 *edx, bool exact_only); |
| 32 | 32 | ||
| 33 | u32 xstate_required_size(u64 xstate_bv, bool compacted); | ||
| 34 | |||
| 33 | int cpuid_query_maxphyaddr(struct kvm_vcpu *vcpu); | 35 | int cpuid_query_maxphyaddr(struct kvm_vcpu *vcpu); |
| 34 | u64 kvm_vcpu_reserved_gpa_bits_raw(struct kvm_vcpu *vcpu); | 36 | u64 kvm_vcpu_reserved_gpa_bits_raw(struct kvm_vcpu *vcpu); |
| 35 | 37 | ||
| @@ -4314,6 +4314,14 @@ int kvm_vm_ioctl_check_extension(struct kvm *kvm, long ext) · 4313 unmodified lines | |||
| 4314 | else | 4314 | else |
| 4315 | r = 0; | 4315 | r = 0; |
| 4316 | break; | 4316 | break; |
| 4317 | case KVM_CAP_XSAVE2: { | ||
| 4318 | u64 guest_perm = xstate_get_guest_group_perm(); | ||
| 4319 | |||
| 4320 | r = xstate_required_size(supported_xcr0 & guest_perm, false); | ||
| 4321 | if (r < sizeof(struct kvm_xsave)) | ||
| 4322 | r = sizeof(struct kvm_xsave); | ||
| 4323 | break; | ||
| 4324 | } | ||
| 4317 | default: | 4325 | default: |
| 4318 | break; | 4326 | break; |
| 4319 | } | 4327 | } |
| @@ -4917,6 +4925,16 @@ static void kvm_vcpu_ioctl_x86_get_xsave(struct kvm_vcpu *vcpu, · 597 unmodified lines | |||
| 4917 | vcpu->arch.pkru); | 4925 | vcpu->arch.pkru); |
| 4918 | } | 4926 | } |
| 4919 | 4927 | ||
| 4928 | static void kvm_vcpu_ioctl_x86_get_xsave2(struct kvm_vcpu *vcpu, | ||
| 4929 | u8 *state, unsigned int size) | ||
| 4930 | { | ||
| 4931 | if (fpstate_is_confidential(&vcpu->arch.guest_fpu)) | ||
| 4932 | return; | ||
| 4933 | |||
| 4934 | fpu_copy_guest_fpstate_to_uabi(&vcpu->arch.guest_fpu, | ||
| 4935 | state, size, vcpu->arch.pkru); | ||
| 4936 | } | ||
| 4937 | |||
| 4920 | static int kvm_vcpu_ioctl_x86_set_xsave(struct kvm_vcpu *vcpu, | 4938 | static int kvm_vcpu_ioctl_x86_set_xsave(struct kvm_vcpu *vcpu, |
| 4921 | struct kvm_xsave *guest_xsave) | 4939 | struct kvm_xsave *guest_xsave) |
| 4922 | { | 4940 | { |
| @@ -5370,6 +5388,10 @@ long kvm_arch_vcpu_ioctl(struct file *filp, · 447 unmodified lines | |||
| 5370 | break; | 5388 | break; |
| 5371 | } | 5389 | } |
| 5372 | case KVM_GET_XSAVE: { | 5390 | case KVM_GET_XSAVE: { |
| 5391 | r = -EINVAL; | ||
| 5392 | if (vcpu->arch.guest_fpu.uabi_size > sizeof(struct kvm_xsave)) | ||
| 5393 | break; | ||
| 5394 | |||
| 5373 | u.xsave = kzalloc(sizeof(struct kvm_xsave), GFP_KERNEL_ACCOUNT); | 5395 | u.xsave = kzalloc(sizeof(struct kvm_xsave), GFP_KERNEL_ACCOUNT); |
| 5374 | r = -ENOMEM; | 5396 | r = -ENOMEM; |
| 5375 | if (!u.xsave) | 5397 | if (!u.xsave) |
| @@ -5384,7 +5406,9 @@ long kvm_arch_vcpu_ioctl(struct file *filp, · 8 unmodified lines | |||
| 5384 | break; | 5406 | break; |
| 5385 | } | 5407 | } |
| 5386 | case KVM_SET_XSAVE: { | 5408 | case KVM_SET_XSAVE: { |
| 5387 | u.xsave = memdup_user(argp, sizeof(*u.xsave)); | 5409 | int size = vcpu->arch.guest_fpu.uabi_size; |
| 5410 | |||
| 5411 | u.xsave = memdup_user(argp, size); | ||
| 5388 | if (IS_ERR(u.xsave)) { | 5412 | if (IS_ERR(u.xsave)) { |
| 5389 | r = PTR_ERR(u.xsave); | 5413 | r = PTR_ERR(u.xsave); |
| 5390 | goto out_nofree; | 5414 | goto out_nofree; |
| @@ -5393,6 +5417,25 @@ long kvm_arch_vcpu_ioctl(struct file *filp, · 2 unmodified lines | |||
| 5393 | r = kvm_vcpu_ioctl_x86_set_xsave(vcpu, u.xsave); | 5417 | r = kvm_vcpu_ioctl_x86_set_xsave(vcpu, u.xsave); |
| 5394 | break; | 5418 | break; |
| 5395 | } | 5419 | } |
| 5420 | |||
| 5421 | case KVM_GET_XSAVE2: { | ||
| 5422 | int size = vcpu->arch.guest_fpu.uabi_size; | ||
| 5423 | |||
| 5424 | u.xsave = kzalloc(size, GFP_KERNEL_ACCOUNT); | ||
| 5425 | r = -ENOMEM; | ||
| 5426 | if (!u.xsave) | ||
| 5427 | break; | ||
| 5428 | |||
| 5429 | kvm_vcpu_ioctl_x86_get_xsave2(vcpu, u.buffer, size); | ||
| 5430 | |||
| 5431 | r = -EFAULT; | ||
| 5432 | if (copy_to_user(argp, u.xsave, size)) | ||
| 5433 | break; | ||
| 5434 | |||
| 5435 | r = 0; | ||
| 5436 | break; | ||
| 5437 | } | ||
| 5438 | |||
| 5396 | case KVM_GET_XCRS: { | 5439 | case KVM_GET_XCRS: { |
| 5397 | u.xcrs = kzalloc(sizeof(struct kvm_xcrs), GFP_KERNEL_ACCOUNT); | 5440 | u.xcrs = kzalloc(sizeof(struct kvm_xcrs), GFP_KERNEL_ACCOUNT); |
| 5398 | r = -ENOMEM; | 5441 | r = -ENOMEM; |
| @@ -1132,6 +1132,7 @@ struct kvm_ppc_resize_hpt { · 1131 unmodified lines | |||
| 1132 | #define KVM_CAP_ARM_MTE 205 | 1132 | #define KVM_CAP_ARM_MTE 205 |
| 1133 | #define KVM_CAP_VM_MOVE_ENC_CONTEXT_FROM 206 | 1133 | #define KVM_CAP_VM_MOVE_ENC_CONTEXT_FROM 206 |
| 1134 | #define KVM_CAP_VM_GPA_BITS 207 | 1134 | #define KVM_CAP_VM_GPA_BITS 207 |
| 1135 | #define KVM_CAP_XSAVE2 208 | ||
| 1135 | 1136 | ||
| 1136 | #ifdef KVM_CAP_IRQ_ROUTING | 1137 | #ifdef KVM_CAP_IRQ_ROUTING |
| 1137 | 1138 | ||
| @@ -1622,6 +1623,9 @@ struct kvm_enc_region { · 484 unmodified lines | |||
| 1622 | #define KVM_S390_NORMAL_RESET _IO(KVMIO, 0xc3) | 1623 | #define KVM_S390_NORMAL_RESET _IO(KVMIO, 0xc3) |
| 1623 | #define KVM_S390_CLEAR_RESET _IO(KVMIO, 0xc4) | 1624 | #define KVM_S390_CLEAR_RESET _IO(KVMIO, 0xc4) |
| 1624 | 1625 | ||
| 1626 | /* Available with KVM_CAP_XSAVE2 */ | ||
| 1627 | #define KVM_GET_XSAVE2 _IOR(KVMIO, 0xcf, struct kvm_xsave) | ||
| 1628 | |||
| 1625 | struct kvm_s390_pv_sec_parm { | 1629 | struct kvm_s390_pv_sec_parm { |
| 1626 | __u64 origin; | 1630 | __u64 origin; |
| 1627 | __u64 length; | 1631 | __u64 length; |