/* SPDX-License-Identifier: GPL-2.0-only *//* * Copyright (C) 2018, Google LLC. */ #ifndef SELFTEST_KVM_PROCESSOR_H#define SELFTEST_KVM_PROCESSOR_H #include <assert.h>#include <stdint.h>#include <syscall.h> #include <asm/msr-index.h>#include <asm/prctl.h> #include <linux/kvm_para.h>#include <linux/stringify.h> #include "kvm_util.h"#include "ucall_common.h" extern bool host_cpu_is_intel;extern bool host_cpu_is_amd;extern bool host_cpu_is_hygon;extern bool host_cpu_is_amd_compatible;extern u64 guest_tsc_khz;extern struct kvm_mmu guest_mmu; #ifndef MAX_NR_CPUID_ENTRIES#define MAX_NR_CPUID_ENTRIES 100#endif #define NONCANONICAL 0xaaaaaaaaaaaaaaaaull /* Forced emulation prefix, used to invoke the emulator unconditionally. */#define KVM_FEP "ud2; .byte 'k', 'v', 'm';" #define NMI_VECTOR 0x02 const char *ex_str(int vector); #define X86_EFLAGS_CF BIT(0) /* Carry Flag */#define X86_EFLAGS_FIXED BIT(1) /* Bit 1 - always on */#define X86_EFLAGS_PF BIT(2) /* Parity Flag */#define X86_EFLAGS_AF BIT(4) /* Auxiliary carry Flag */#define X86_EFLAGS_ZF BIT(6) /* Zero Flag */#define X86_EFLAGS_SF BIT(7) /* Sign Flag */#define X86_EFLAGS_TF BIT(8) /* Trap Flag */#define X86_EFLAGS_IF BIT(9) /* Interrupt Flag */#define X86_EFLAGS_DF BIT(10) /* Direction Flag */#define X86_EFLAGS_OF BIT(11) /* Overflow Flag */#define X86_EFLAGS_IOPL BIT(12) /* I/O Privilege Level (2 bits) */#define X86_EFLAGS_NT BIT(14) /* Nested Task */#define X86_EFLAGS_RF BIT(16) /* Resume Flag */#define X86_EFLAGS_VM BIT(17) /* Virtual Mode */#define X86_EFLAGS_AC BIT(18) /* Alignment Check/Access Control */#define X86_EFLAGS_VIF BIT(19) /* Virtual Interrupt Flag */#define X86_EFLAGS_VIP BIT(20) /* Virtual Interrupt Pending */#define X86_EFLAGS_ID BIT(21) /* CPUID detection */ #define X86_CR4_VME (1ul << 0)#define X86_CR4_PVI (1ul << 1)#define X86_CR4_TSD (1ul << 2)#define X86_CR4_DE (1ul << 3)#define X86_CR4_PSE (1ul << 4)#define X86_CR4_PAE (1ul << 5)#define X86_CR4_MCE (1ul << 6)#define X86_CR4_PGE (1ul << 7)#define X86_CR4_PCE (1ul << 8)#define X86_CR4_OSFXSR (1ul << 9)#define X86_CR4_OSXMMEXCPT (1ul << 10)#define X86_CR4_UMIP (1ul << 11)#define X86_CR4_LA57 (1ul << 12)#define X86_CR4_VMXE (1ul << 13)#define X86_CR4_SMXE (1ul << 14)#define X86_CR4_FSGSBASE (1ul << 16)#define X86_CR4_PCIDE (1ul << 17)#define X86_CR4_OSXSAVE (1ul << 18)#define X86_CR4_SMEP (1ul << 20)#define X86_CR4_SMAP (1ul << 21)#define X86_CR4_PKE (1ul << 22) struct xstate_header { u64 xstate_bv; u64 xcomp_bv; u64 reserved[6];} __attribute__((packed)); struct xstate { u8 i387[512]; struct xstate_header header; u8 extended_state_area[0];} __attribute__ ((packed, aligned (64))); #define XFEATURE_MASK_FP BIT_ULL(0)#define XFEATURE_MASK_SSE BIT_ULL(1)#define XFEATURE_MASK_YMM BIT_ULL(2)#define XFEATURE_MASK_BNDREGS BIT_ULL(3)#define XFEATURE_MASK_BNDCSR BIT_ULL(4)#define XFEATURE_MASK_OPMASK BIT_ULL(5)#define XFEATURE_MASK_ZMM_Hi256 BIT_ULL(6)#define XFEATURE_MASK_Hi16_ZMM BIT_ULL(7)#define XFEATURE_MASK_PT BIT_ULL(8)#define XFEATURE_MASK_PKRU BIT_ULL(9)#define XFEATURE_MASK_PASID BIT_ULL(10)#define XFEATURE_MASK_CET_USER BIT_ULL(11)#define XFEATURE_MASK_CET_KERNEL BIT_ULL(12)#define XFEATURE_MASK_LBR BIT_ULL(15)#define XFEATURE_MASK_XTILE_CFG BIT_ULL(17)#define XFEATURE_MASK_XTILE_DATA BIT_ULL(18) #define XFEATURE_MASK_AVX512 (XFEATURE_MASK_OPMASK | \ XFEATURE_MASK_ZMM_Hi256 | \ XFEATURE_MASK_Hi16_ZMM)#define XFEATURE_MASK_XTILE (XFEATURE_MASK_XTILE_DATA | \ XFEATURE_MASK_XTILE_CFG) /* Note, these are ordered alphabetically to match kvm_cpuid_entry2. Eww. */enum cpuid_output_regs { KVM_CPUID_EAX, KVM_CPUID_EBX, KVM_CPUID_ECX, KVM_CPUID_EDX}; /* * Pack the information into a 64-bit value so that each X86_FEATURE_XXX can be * passed by value with no overhead. */struct kvm_x86_cpu_feature { u32 function; u16 index; u8 reg; u8 bit;};#define KVM_X86_CPU_FEATURE(fn, idx, gpr, __bit) \({ \ struct kvm_x86_cpu_feature feature = { \ .function = fn, \ .index = idx, \ .reg = KVM_CPUID_##gpr, \ .bit = __bit, \ }; \ \ kvm_static_assert((fn & 0xc0000000) == 0 || \ (fn & 0xc0000000) == 0x40000000 || \ (fn & 0xc0000000) == 0x80000000 || \ (fn & 0xc0000000) == 0xc0000000); \ kvm_static_assert(idx < BIT(sizeof(feature.index) * BITS_PER_BYTE)); \ feature; \}) /* * Basic Leafs, a.k.a. Intel defined */#define X86_FEATURE_MWAIT KVM_X86_CPU_FEATURE(0x1, 0, ECX, 3)#define X86_FEATURE_VMX KVM_X86_CPU_FEATURE(0x1, 0, ECX, 5)#define X86_FEATURE_SMX KVM_X86_CPU_FEATURE(0x1, 0, ECX, 6)#define X86_FEATURE_PDCM KVM_X86_CPU_FEATURE(0x1, 0, ECX, 15)#define X86_FEATURE_PCID KVM_X86_CPU_FEATURE(0x1, 0, ECX, 17)#define X86_FEATURE_X2APIC KVM_X86_CPU_FEATURE(0x1, 0, ECX, 21)#define X86_FEATURE_MOVBE KVM_X86_CPU_FEATURE(0x1, 0, ECX, 22)#define X86_FEATURE_TSC_DEADLINE_TIMER KVM_X86_CPU_FEATURE(0x1, 0, ECX, 24)#define X86_FEATURE_XSAVE KVM_X86_CPU_FEATURE(0x1, 0, ECX, 26)#define X86_FEATURE_OSXSAVE KVM_X86_CPU_FEATURE(0x1, 0, ECX, 27)#define X86_FEATURE_RDRAND KVM_X86_CPU_FEATURE(0x1, 0, ECX, 30)#define X86_FEATURE_HYPERVISOR KVM_X86_CPU_FEATURE(0x1, 0, ECX, 31)#define X86_FEATURE_PAE KVM_X86_CPU_FEATURE(0x1, 0, EDX, 6)#define X86_FEATURE_MCE KVM_X86_CPU_FEATURE(0x1, 0, EDX, 7)#define X86_FEATURE_APIC KVM_X86_CPU_FEATURE(0x1, 0, EDX, 9)#define X86_FEATURE_CLFLUSH KVM_X86_CPU_FEATURE(0x1, 0, EDX, 19)#define X86_FEATURE_XMM KVM_X86_CPU_FEATURE(0x1, 0, EDX, 25)#define X86_FEATURE_XMM2 KVM_X86_CPU_FEATURE(0x1, 0, EDX, 26)#define X86_FEATURE_FSGSBASE KVM_X86_CPU_FEATURE(0x7, 0, EBX, 0)#define X86_FEATURE_TSC_ADJUST KVM_X86_CPU_FEATURE(0x7, 0, EBX, 1)#define X86_FEATURE_SGX KVM_X86_CPU_FEATURE(0x7, 0, EBX, 2)#define X86_FEATURE_HLE KVM_X86_CPU_FEATURE(0x7, 0, EBX, 4)#define X86_FEATURE_SMEP KVM_X86_CPU_FEATURE(0x7, 0, EBX, 7)#define X86_FEATURE_INVPCID KVM_X86_CPU_FEATURE(0x7, 0, EBX, 10)#define X86_FEATURE_RTM KVM_X86_CPU_FEATURE(0x7, 0, EBX, 11)#define X86_FEATURE_MPX KVM_X86_CPU_FEATURE(0x7, 0, EBX, 14)#define X86_FEATURE_SMAP KVM_X86_CPU_FEATURE(0x7, 0, EBX, 20)#define X86_FEATURE_PCOMMIT KVM_X86_CPU_FEATURE(0x7, 0, EBX, 22)#define X86_FEATURE_CLFLUSHOPT KVM_X86_CPU_FEATURE(0x7, 0, EBX, 23)#define X86_FEATURE_CLWB KVM_X86_CPU_FEATURE(0x7, 0, EBX, 24)#define X86_FEATURE_UMIP KVM_X86_CPU_FEATURE(0x7, 0, ECX, 2)#define X86_FEATURE_PKU KVM_X86_CPU_FEATURE(0x7, 0, ECX, 3)#define X86_FEATURE_OSPKE KVM_X86_CPU_FEATURE(0x7, 0, ECX, 4)#define X86_FEATURE_LA57 KVM_X86_CPU_FEATURE(0x7, 0, ECX, 16)#define X86_FEATURE_RDPID KVM_X86_CPU_FEATURE(0x7, 0, ECX, 22)#define X86_FEATURE_SGX_LC KVM_X86_CPU_FEATURE(0x7, 0, ECX, 30)#define X86_FEATURE_SHSTK KVM_X86_CPU_FEATURE(0x7, 0, ECX, 7)#define X86_FEATURE_IBT KVM_X86_CPU_FEATURE(0x7, 0, EDX, 20)#define X86_FEATURE_AMX_TILE KVM_X86_CPU_FEATURE(0x7, 0, EDX, 24)#define X86_FEATURE_SPEC_CTRL KVM_X86_CPU_FEATURE(0x7, 0, EDX, 26)#define X86_FEATURE_ARCH_CAPABILITIES KVM_X86_CPU_FEATURE(0x7, 0, EDX, 29)#define X86_FEATURE_PKS KVM_X86_CPU_FEATURE(0x7, 0, ECX, 31)#define X86_FEATURE_XTILECFG KVM_X86_CPU_FEATURE(0xD, 0, EAX, 17)#define X86_FEATURE_XTILEDATA KVM_X86_CPU_FEATURE(0xD, 0, EAX, 18)#define X86_FEATURE_XSAVES KVM_X86_CPU_FEATURE(0xD, 1, EAX, 3)#define X86_FEATURE_XFD KVM_X86_CPU_FEATURE(0xD, 1, EAX, 4)#define X86_FEATURE_XTILEDATA_XFD KVM_X86_CPU_FEATURE(0xD, 18, ECX, 2) /* * Extended Leafs, a.k.a. AMD defined */#define X86_FEATURE_SVM KVM_X86_CPU_FEATURE(0x80000001, 0, ECX, 2)#define X86_FEATURE_PERFCTR_CORE KVM_X86_CPU_FEATURE(0x80000001, 0, ECX, 23)#define X86_FEATURE_PERFCTR_NB KVM_X86_CPU_FEATURE(0x80000001, 0, ECX, 24)#define X86_FEATURE_PERFCTR_LLC KVM_X86_CPU_FEATURE(0x80000001, 0, ECX, 28)#define X86_FEATURE_NX KVM_X86_CPU_FEATURE(0x80000001, 0, EDX, 20)#define X86_FEATURE_FXSR_OPT KVM_X86_CPU_FEATURE(0x80000001, 0, EDX, 25)#define X86_FEATURE_GBPAGES KVM_X86_CPU_FEATURE(0x80000001, 0, EDX, 26)#define X86_FEATURE_RDTSCP KVM_X86_CPU_FEATURE(0x80000001, 0, EDX, 27)#define X86_FEATURE_LM KVM_X86_CPU_FEATURE(0x80000001, 0, EDX, 29)#define X86_FEATURE_INVTSC KVM_X86_CPU_FEATURE(0x80000007, 0, EDX, 8)#define X86_FEATURE_RDPRU KVM_X86_CPU_FEATURE(0x80000008, 0, EBX, 4)#define X86_FEATURE_AMD_IBPB KVM_X86_CPU_FEATURE(0x80000008, 0, EBX, 12)#define X86_FEATURE_NPT KVM_X86_CPU_FEATURE(0x8000000A, 0, EDX, 0)#define X86_FEATURE_LBRV KVM_X86_CPU_FEATURE(0x8000000A, 0, EDX, 1)#define X86_FEATURE_NRIPS KVM_X86_CPU_FEATURE(0x8000000A, 0, EDX, 3)#define X86_FEATURE_TSCRATEMSR KVM_X86_CPU_FEATURE(0x8000000A, 0, EDX, 4)#define X86_FEATURE_PAUSEFILTER KVM_X86_CPU_FEATURE(0x8000000A, 0, EDX, 10)#define X86_FEATURE_PFTHRESHOLD KVM_X86_CPU_FEATURE(0x8000000A, 0, EDX, 12)#define X86_FEATURE_V_VMSAVE_VMLOAD KVM_X86_CPU_FEATURE(0x8000000A, 0, EDX, 15)#define X86_FEATURE_VGIF KVM_X86_CPU_FEATURE(0x8000000A, 0, EDX, 16)#define X86_FEATURE_IDLE_HLT KVM_X86_CPU_FEATURE(0x8000000A, 0, EDX, 30)#define X86_FEATURE_SEV KVM_X86_CPU_FEATURE(0x8000001F, 0, EAX, 1)#define X86_FEATURE_SEV_ES KVM_X86_CPU_FEATURE(0x8000001F, 0, EAX, 3)#define X86_FEATURE_SEV_SNP KVM_X86_CPU_FEATURE(0x8000001F, 0, EAX, 4)#define X86_FEATURE_AUTOIBRS KVM_X86_CPU_FEATURE(0x80000021, 0, EAX, 8)#define X86_FEATURE_GP_ON_USER_CPUID KVM_X86_CPU_FEATURE(0x80000021, 0, EAX, 17)#define X86_FEATURE_PERFMON_V2 KVM_X86_CPU_FEATURE(0x80000022, 0, EAX, 0)#define X86_FEATURE_LBR_PMC_FREEZE KVM_X86_CPU_FEATURE(0x80000022, 0, EAX, 2) /* * KVM defined paravirt features. */#define X86_FEATURE_KVM_CLOCKSOURCE KVM_X86_CPU_FEATURE(0x40000001, 0, EAX, 0)#define X86_FEATURE_KVM_NOP_IO_DELAY KVM_X86_CPU_FEATURE(0x40000001, 0, EAX, 1)#define X86_FEATURE_KVM_MMU_OP KVM_X86_CPU_FEATURE(0x40000001, 0, EAX, 2)#define X86_FEATURE_KVM_CLOCKSOURCE2 KVM_X86_CPU_FEATURE(0x40000001, 0, EAX, 3)#define X86_FEATURE_KVM_ASYNC_PF KVM_X86_CPU_FEATURE(0x40000001, 0, EAX, 4)#define X86_FEATURE_KVM_STEAL_TIME KVM_X86_CPU_FEATURE(0x40000001, 0, EAX, 5)#define X86_FEATURE_KVM_PV_EOI KVM_X86_CPU_FEATURE(0x40000001, 0, EAX, 6)#define X86_FEATURE_KVM_PV_UNHALT KVM_X86_CPU_FEATURE(0x40000001, 0, EAX, 7)/* Bit 8 apparently isn't used?!?! */#define X86_FEATURE_KVM_PV_TLB_FLUSH KVM_X86_CPU_FEATURE(0x40000001, 0, EAX, 9)#define X86_FEATURE_KVM_ASYNC_PF_VMEXIT KVM_X86_CPU_FEATURE(0x40000001, 0, EAX, 10)#define X86_FEATURE_KVM_PV_SEND_IPI KVM_X86_CPU_FEATURE(0x40000001, 0, EAX, 11)#define X86_FEATURE_KVM_POLL_CONTROL KVM_X86_CPU_FEATURE(0x40000001, 0, EAX, 12)#define X86_FEATURE_KVM_PV_SCHED_YIELD KVM_X86_CPU_FEATURE(0x40000001, 0, EAX, 13)#define X86_FEATURE_KVM_ASYNC_PF_INT KVM_X86_CPU_FEATURE(0x40000001, 0, EAX, 14)#define X86_FEATURE_KVM_MSI_EXT_DEST_ID KVM_X86_CPU_FEATURE(0x40000001, 0, EAX, 15)#define X86_FEATURE_KVM_HC_MAP_GPA_RANGE KVM_X86_CPU_FEATURE(0x40000001, 0, EAX, 16)#define X86_FEATURE_KVM_MIGRATION_CONTROL KVM_X86_CPU_FEATURE(0x40000001, 0, EAX, 17) /* * Same idea as X86_FEATURE_XXX, but X86_PROPERTY_XXX retrieves a multi-bit * value/property as opposed to a single-bit feature. Again, pack the info * into a 64-bit value to pass by value with no overhead. */struct kvm_x86_cpu_property { u32 function; u8 index; u8 reg; u8 lo_bit; u8 hi_bit;};#define KVM_X86_CPU_PROPERTY(fn, idx, gpr, low_bit, high_bit) \({ \ struct kvm_x86_cpu_property property = { \ .function = fn, \ .index = idx, \ .reg = KVM_CPUID_##gpr, \ .lo_bit = low_bit, \ .hi_bit = high_bit, \ }; \ \ kvm_static_assert(low_bit < high_bit); \ kvm_static_assert((fn & 0xc0000000) == 0 || \ (fn & 0xc0000000) == 0x40000000 || \ (fn & 0xc0000000) == 0x80000000 || \ (fn & 0xc0000000) == 0xc0000000); \ kvm_static_assert(idx < BIT(sizeof(property.index) * BITS_PER_BYTE)); \ property; \}) #define X86_PROPERTY_MAX_BASIC_LEAF KVM_X86_CPU_PROPERTY(0, 0, EAX, 0, 31)#define X86_PROPERTY_PMU_VERSION KVM_X86_CPU_PROPERTY(0xa, 0, EAX, 0, 7)#define X86_PROPERTY_PMU_NR_GP_COUNTERS KVM_X86_CPU_PROPERTY(0xa, 0, EAX, 8, 15)#define X86_PROPERTY_PMU_GP_COUNTERS_BIT_WIDTH KVM_X86_CPU_PROPERTY(0xa, 0, EAX, 16, 23)#define X86_PROPERTY_PMU_EBX_BIT_VECTOR_LENGTH KVM_X86_CPU_PROPERTY(0xa, 0, EAX, 24, 31)#define X86_PROPERTY_PMU_EVENTS_MASK KVM_X86_CPU_PROPERTY(0xa, 0, EBX, 0, 12)#define X86_PROPERTY_PMU_FIXED_COUNTERS_BITMASK KVM_X86_CPU_PROPERTY(0xa, 0, ECX, 0, 31)#define X86_PROPERTY_PMU_NR_FIXED_COUNTERS KVM_X86_CPU_PROPERTY(0xa, 0, EDX, 0, 4)#define X86_PROPERTY_PMU_FIXED_COUNTERS_BIT_WIDTH KVM_X86_CPU_PROPERTY(0xa, 0, EDX, 5, 12) #define X86_PROPERTY_SUPPORTED_XCR0_LO KVM_X86_CPU_PROPERTY(0xd, 0, EAX, 0, 31)#define X86_PROPERTY_XSTATE_MAX_SIZE_XCR0 KVM_X86_CPU_PROPERTY(0xd, 0, EBX, 0, 31)#define X86_PROPERTY_XSTATE_MAX_SIZE KVM_X86_CPU_PROPERTY(0xd, 0, ECX, 0, 31)#define X86_PROPERTY_SUPPORTED_XCR0_HI KVM_X86_CPU_PROPERTY(0xd, 0, EDX, 0, 31) #define X86_PROPERTY_XSTATE_TILE_SIZE KVM_X86_CPU_PROPERTY(0xd, 18, EAX, 0, 31)#define X86_PROPERTY_XSTATE_TILE_OFFSET KVM_X86_CPU_PROPERTY(0xd, 18, EBX, 0, 31)#define X86_PROPERTY_AMX_MAX_PALETTE_TABLES KVM_X86_CPU_PROPERTY(0x1d, 0, EAX, 0, 31)#define X86_PROPERTY_AMX_TOTAL_TILE_BYTES KVM_X86_CPU_PROPERTY(0x1d, 1, EAX, 0, 15)#define X86_PROPERTY_AMX_BYTES_PER_TILE KVM_X86_CPU_PROPERTY(0x1d, 1, EAX, 16, 31)#define X86_PROPERTY_AMX_BYTES_PER_ROW KVM_X86_CPU_PROPERTY(0x1d, 1, EBX, 0, 15)#define X86_PROPERTY_AMX_NR_TILE_REGS KVM_X86_CPU_PROPERTY(0x1d, 1, EBX, 16, 31)#define X86_PROPERTY_AMX_MAX_ROWS KVM_X86_CPU_PROPERTY(0x1d, 1, ECX, 0, 15) #define X86_PROPERTY_MAX_KVM_LEAF KVM_X86_CPU_PROPERTY(0x40000000, 0, EAX, 0, 31) #define X86_PROPERTY_MAX_EXT_LEAF KVM_X86_CPU_PROPERTY(0x80000000, 0, EAX, 0, 31)#define X86_PROPERTY_MAX_PHY_ADDR KVM_X86_CPU_PROPERTY(0x80000008, 0, EAX, 0, 7)#define X86_PROPERTY_MAX_VIRT_ADDR KVM_X86_CPU_PROPERTY(0x80000008, 0, EAX, 8, 15)#define X86_PROPERTY_GUEST_MAX_PHY_ADDR KVM_X86_CPU_PROPERTY(0x80000008, 0, EAX, 16, 23)#define X86_PROPERTY_SEV_C_BIT KVM_X86_CPU_PROPERTY(0x8000001F, 0, EBX, 0, 5)#define X86_PROPERTY_PHYS_ADDR_REDUCTION KVM_X86_CPU_PROPERTY(0x8000001F, 0, EBX, 6, 11)#define X86_PROPERTY_NR_PERFCTR_CORE KVM_X86_CPU_PROPERTY(0x80000022, 0, EBX, 0, 3)#define X86_PROPERTY_NR_PERFCTR_NB KVM_X86_CPU_PROPERTY(0x80000022, 0, EBX, 10, 15) #define X86_PROPERTY_MAX_CENTAUR_LEAF KVM_X86_CPU_PROPERTY(0xC0000000, 0, EAX, 0, 31) /* * Intel's architectural PMU events are bizarre. They have a "feature" bit * that indicates the feature is _not_ supported, and a property that states * the length of the bit mask of unsupported features. A feature is supported * if the size of the bit mask is larger than the "unavailable" bit, and said * bit is not set. Fixed counters also bizarre enumeration, but inverted from * arch events for general purpose counters. Fixed counters are supported if a * feature flag is set **OR** the total number of fixed counters is greater * than index of the counter. * * Wrap the events for general purpose and fixed counters to simplify checking * whether or not a given architectural event is supported. */struct kvm_x86_pmu_feature { struct kvm_x86_cpu_feature f;};#define KVM_X86_PMU_FEATURE(__reg, __bit) \({ \ struct kvm_x86_pmu_feature feature = { \ .f = KVM_X86_CPU_FEATURE(0xa, 0, __reg, __bit), \ }; \ \ kvm_static_assert(KVM_CPUID_##__reg == KVM_CPUID_EBX || \ KVM_CPUID_##__reg == KVM_CPUID_ECX); \ feature; \}) #define X86_PMU_FEATURE_CPU_CYCLES KVM_X86_PMU_FEATURE(EBX, 0)#define X86_PMU_FEATURE_INSNS_RETIRED KVM_X86_PMU_FEATURE(EBX, 1)#define X86_PMU_FEATURE_REFERENCE_CYCLES KVM_X86_PMU_FEATURE(EBX, 2)#define X86_PMU_FEATURE_LLC_REFERENCES KVM_X86_PMU_FEATURE(EBX, 3)#define X86_PMU_FEATURE_LLC_MISSES KVM_X86_PMU_FEATURE(EBX, 4)#define X86_PMU_FEATURE_BRANCH_INSNS_RETIRED KVM_X86_PMU_FEATURE(EBX, 5)#define X86_PMU_FEATURE_BRANCHES_MISPREDICTED KVM_X86_PMU_FEATURE(EBX, 6)#define X86_PMU_FEATURE_TOPDOWN_SLOTS KVM_X86_PMU_FEATURE(EBX, 7)#define X86_PMU_FEATURE_TOPDOWN_BE_BOUND KVM_X86_PMU_FEATURE(EBX, 8)#define X86_PMU_FEATURE_TOPDOWN_BAD_SPEC KVM_X86_PMU_FEATURE(EBX, 9)#define X86_PMU_FEATURE_TOPDOWN_FE_BOUND KVM_X86_PMU_FEATURE(EBX, 10)#define X86_PMU_FEATURE_TOPDOWN_RETIRING KVM_X86_PMU_FEATURE(EBX, 11)#define X86_PMU_FEATURE_LBR_INSERTS KVM_X86_PMU_FEATURE(EBX, 12) #define X86_PMU_FEATURE_INSNS_RETIRED_FIXED KVM_X86_PMU_FEATURE(ECX, 0)#define X86_PMU_FEATURE_CPU_CYCLES_FIXED KVM_X86_PMU_FEATURE(ECX, 1)#define X86_PMU_FEATURE_REFERENCE_TSC_CYCLES_FIXED KVM_X86_PMU_FEATURE(ECX, 2)#define X86_PMU_FEATURE_TOPDOWN_SLOTS_FIXED KVM_X86_PMU_FEATURE(ECX, 3) static inline unsigned int x86_family(unsigned int eax){ unsigned int x86; x86 = (eax >> 8) & 0xf; if (x86 == 0xf) x86 += (eax >> 20) & 0xff; return x86;} static inline unsigned int x86_model(unsigned int eax){ return ((eax >> 12) & 0xf0) | ((eax >> 4) & 0x0f);} #define PHYSICAL_PAGE_MASK GENMASK_ULL(51, 12) #define PAGE_SHIFT 12#define PAGE_SIZE (1ULL << PAGE_SHIFT)#define PAGE_MASK (~(PAGE_SIZE-1) & PHYSICAL_PAGE_MASK) #define HUGEPAGE_SHIFT(x) (PAGE_SHIFT + (((x) - 1) * 9))#define HUGEPAGE_SIZE(x) (1UL << HUGEPAGE_SHIFT(x))#define HUGEPAGE_MASK(x) (~(HUGEPAGE_SIZE(x) - 1) & PHYSICAL_PAGE_MASK) #define PTE_GET_PA(pte) ((pte) & PHYSICAL_PAGE_MASK)#define PTE_GET_PFN(pte) (PTE_GET_PA(pte) >> PAGE_SHIFT) struct guest_regs { u64 rax; u64 rcx; u64 rdx; u64 rbx; u64 rsp; u64 rbp; u64 rsi; u64 rdi; u64 r8; u64 r9; u64 r10; u64 r11; u64 r12; u64 r13; u64 r14; u64 r15; u64 rflags;}; extern struct guest_regs guest_regs; #define GUEST_REG_OFFSET(name) \ [off_##name] "i" (offsetof(struct guest_regs, name)) #define GUEST_REGS_OFFSETS \ GUEST_REG_OFFSET(rax), \ GUEST_REG_OFFSET(rcx), \ GUEST_REG_OFFSET(rdx), \ GUEST_REG_OFFSET(rbx), \ GUEST_REG_OFFSET(rsp), \ GUEST_REG_OFFSET(rbp), \ GUEST_REG_OFFSET(rsi), \ GUEST_REG_OFFSET(rdi), \ GUEST_REG_OFFSET(r8), \ GUEST_REG_OFFSET(r9), \ GUEST_REG_OFFSET(r10), \ GUEST_REG_OFFSET(r11), \ GUEST_REG_OFFSET(r12), \ GUEST_REG_OFFSET(r13), \ GUEST_REG_OFFSET(r14), \ GUEST_REG_OFFSET(r15), \ GUEST_REG_OFFSET(rflags) #define GUEST_REG(name) "guest_regs + %c[off_" #name "]" #define GUEST_SWITCH_GPR_ASM(name) \ "xchg %%" #name ", " GUEST_REG(name) "\n\t" struct desc64 { u16 limit0; u16 base0; unsigned base1:8, type:4, s:1, dpl:2, p:1; unsigned limit1:4, avl:1, l:1, db:1, g:1, base2:8; u32 base3; u32 zero1;} __attribute__((packed)); struct desc_ptr { u16 size; u64 address;} __attribute__((packed)); struct kvm_x86_state { struct kvm_xsave *xsave; struct kvm_vcpu_events events; struct kvm_mp_state mp_state; struct kvm_regs regs; struct kvm_xcrs xcrs; struct kvm_sregs sregs; struct kvm_debugregs debugregs; union { struct kvm_nested_state nested; char nested_[16384]; }; struct kvm_msrs msrs;}; static inline u64 get_desc64_base(const struct desc64 *desc){ return (u64)desc->base3 << 32 | (u64)desc->base2 << 24 | (u64)desc->base1 << 16 | (u64)desc->base0;} static inline u64 rdtsc(void){ u32 eax, edx; u64 tsc_val; /* * The lfence is to wait (on Intel CPUs) until all previous * instructions have been executed. If software requires RDTSC to be * executed prior to execution of any subsequent instruction, it can * execute LFENCE immediately after RDTSC */ __asm__ __volatile__("lfence; rdtsc; lfence" : "=a"(eax), "=d"(edx)); tsc_val = ((u64)edx) << 32 | eax; return tsc_val;Lines 1–500 of 1648. Load more
/* SPDX-License-Identifier: GPL-2.0-only */ /* * Copyright (C) 2018, Google LLC. */ #ifndef SELFTEST_KVM_PROCESSOR_H #define SELFTEST_KVM_PROCESSOR_H #include <assert.h> #include <stdint.h> #include <syscall.h> #include <asm/msr-index.h> #include <asm/prctl.h> #include <linux/kvm_para.h> #include <linux/stringify.h> #include "kvm_util.h" #include "ucall_common.h" extern bool host_cpu_is_intel; extern bool host_cpu_is_amd; extern bool host_cpu_is_hygon; extern bool host_cpu_is_amd_compatible; extern u64 guest_tsc_khz; extern struct kvm_mmu guest_mmu; #ifndef MAX_NR_CPUID_ENTRIES #define MAX_NR_CPUID_ENTRIES 100 #endif #define NONCANONICAL 0xaaaaaaaaaaaaaaaaull /* Forced emulation prefix, used to invoke the emulator unconditionally. */ #define KVM_FEP "ud2; .byte 'k', 'v', 'm';" #define NMI_VECTOR 0x02 const char *ex_str(int vector); #define X86_EFLAGS_CF BIT(0) /* Carry Flag */ #define X86_EFLAGS_FIXED BIT(1) /* Bit 1 - always on */ #define X86_EFLAGS_PF BIT(2) /* Parity Flag */ #define X86_EFLAGS_AF BIT(4) /* Auxiliary carry Flag */ #define X86_EFLAGS_ZF BIT(6) /* Zero Flag */ #define X86_EFLAGS_SF BIT(7) /* Sign Flag */ #define X86_EFLAGS_TF BIT(8) /* Trap Flag */ #define X86_EFLAGS_IF BIT(9) /* Interrupt Flag */ #define X86_EFLAGS_DF BIT(10) /* Direction Flag */ #define X86_EFLAGS_OF BIT(11) /* Overflow Flag */ #define X86_EFLAGS_IOPL BIT(12) /* I/O Privilege Level (2 bits) */ #define X86_EFLAGS_NT BIT(14) /* Nested Task */ #define X86_EFLAGS_RF BIT(16) /* Resume Flag */ #define X86_EFLAGS_VM BIT(17) /* Virtual Mode */ #define X86_EFLAGS_AC BIT(18) /* Alignment Check/Access Control */ #define X86_EFLAGS_VIF BIT(19) /* Virtual Interrupt Flag */ #define X86_EFLAGS_VIP BIT(20) /* Virtual Interrupt Pending */ #define X86_EFLAGS_ID BIT(21) /* CPUID detection */ #define X86_CR4_VME (1ul << 0) #define X86_CR4_PVI (1ul << 1) #define X86_CR4_TSD (1ul << 2) #define X86_CR4_DE (1ul << 3) #define X86_CR4_PSE (1ul << 4) #define X86_CR4_PAE (1ul << 5) #define X86_CR4_MCE (1ul << 6) #define X86_CR4_PGE (1ul << 7) #define X86_CR4_PCE (1ul << 8) #define X86_CR4_OSFXSR (1ul << 9) #define X86_CR4_OSXMMEXCPT (1ul << 10) #define X86_CR4_UMIP (1ul << 11) #define X86_CR4_LA57 (1ul << 12) #define X86_CR4_VMXE (1ul << 13) #define X86_CR4_SMXE (1ul << 14) #define X86_CR4_FSGSBASE (1ul << 16) #define X86_CR4_PCIDE (1ul << 17) #define X86_CR4_OSXSAVE (1ul << 18) #define X86_CR4_SMEP (1ul << 20) #define X86_CR4_SMAP (1ul << 21) #define X86_CR4_PKE (1ul << 22) struct xstate_header { u64 xstate_bv; u64 xcomp_bv; u64 reserved[6]; } __attribute__((packed)); struct xstate { u8 i387[512]; struct xstate_header header; u8 extended_state_area[0]; } __attribute__ ((packed, aligned (64))); #define XFEATURE_MASK_FP BIT_ULL(0) #define XFEATURE_MASK_SSE BIT_ULL(1) #define XFEATURE_MASK_YMM BIT_ULL(2) #define XFEATURE_MASK_BNDREGS BIT_ULL(3) #define XFEATURE_MASK_BNDCSR BIT_ULL(4) #define XFEATURE_MASK_OPMASK BIT_ULL(5) #define XFEATURE_MASK_ZMM_Hi256 BIT_ULL(6) #define XFEATURE_MASK_Hi16_ZMM BIT_ULL(7) #define XFEATURE_MASK_PT BIT_ULL(8) #define XFEATURE_MASK_PKRU BIT_ULL(9) #define XFEATURE_MASK_PASID BIT_ULL(10) #define XFEATURE_MASK_CET_USER BIT_ULL(11) #define XFEATURE_MASK_CET_KERNEL BIT_ULL(12) #define XFEATURE_MASK_LBR BIT_ULL(15) #define XFEATURE_MASK_XTILE_CFG BIT_ULL(17) #define XFEATURE_MASK_XTILE_DATA BIT_ULL(18) #define XFEATURE_MASK_AVX512 (XFEATURE_MASK_OPMASK | \ XFEATURE_MASK_ZMM_Hi256 | \ XFEATURE_MASK_Hi16_ZMM) #define XFEATURE_MASK_XTILE (XFEATURE_MASK_XTILE_DATA | \ XFEATURE_MASK_XTILE_CFG) /* Note, these are ordered alphabetically to match kvm_cpuid_entry2. Eww. */ enum cpuid_output_regs { KVM_CPUID_EAX, KVM_CPUID_EBX, KVM_CPUID_ECX, KVM_CPUID_EDX }; /* * Pack the information into a 64-bit value so that each X86_FEATURE_XXX can be * passed by value with no overhead. */ struct kvm_x86_cpu_feature { u32 function; u16 index; u8 reg; u8 bit; }; #define KVM_X86_CPU_FEATURE(fn, idx, gpr, __bit) \ ({ \ struct kvm_x86_cpu_feature feature = { \ .function = fn, \ .index = idx, \ .reg = KVM_CPUID_##gpr, \ .bit = __bit, \ }; \ \ kvm_static_assert((fn & 0xc0000000) == 0 || \ (fn & 0xc0000000) == 0x40000000 || \ (fn & 0xc0000000) == 0x80000000 || \ (fn & 0xc0000000) == 0xc0000000); \ kvm_static_assert(idx < BIT(sizeof(feature.index) * BITS_PER_BYTE)); \ feature; \ }) /* * Basic Leafs, a.k.a. Intel defined */ #define X86_FEATURE_MWAIT KVM_X86_CPU_FEATURE(0x1, 0, ECX, 3) #define X86_FEATURE_VMX KVM_X86_CPU_FEATURE(0x1, 0, ECX, 5) #define X86_FEATURE_SMX KVM_X86_CPU_FEATURE(0x1, 0, ECX, 6) #define X86_FEATURE_PDCM KVM_X86_CPU_FEATURE(0x1, 0, ECX, 15) #define X86_FEATURE_PCID KVM_X86_CPU_FEATURE(0x1, 0, ECX, 17) #define X86_FEATURE_X2APIC KVM_X86_CPU_FEATURE(0x1, 0, ECX, 21) #define X86_FEATURE_MOVBE KVM_X86_CPU_FEATURE(0x1, 0, ECX, 22) #define X86_FEATURE_TSC_DEADLINE_TIMER KVM_X86_CPU_FEATURE(0x1, 0, ECX, 24) #define X86_FEATURE_XSAVE KVM_X86_CPU_FEATURE(0x1, 0, ECX, 26) #define X86_FEATURE_OSXSAVE KVM_X86_CPU_FEATURE(0x1, 0, ECX, 27) #define X86_FEATURE_RDRAND KVM_X86_CPU_FEATURE(0x1, 0, ECX, 30) #define X86_FEATURE_HYPERVISOR KVM_X86_CPU_FEATURE(0x1, 0, ECX, 31) #define X86_FEATURE_PAE KVM_X86_CPU_FEATURE(0x1, 0, EDX, 6) #define X86_FEATURE_MCE KVM_X86_CPU_FEATURE(0x1, 0, EDX, 7) #define X86_FEATURE_APIC KVM_X86_CPU_FEATURE(0x1, 0, EDX, 9) #define X86_FEATURE_CLFLUSH KVM_X86_CPU_FEATURE(0x1, 0, EDX, 19) #define X86_FEATURE_XMM KVM_X86_CPU_FEATURE(0x1, 0, EDX, 25) #define X86_FEATURE_XMM2 KVM_X86_CPU_FEATURE(0x1, 0, EDX, 26) #define X86_FEATURE_FSGSBASE KVM_X86_CPU_FEATURE(0x7, 0, EBX, 0) #define X86_FEATURE_TSC_ADJUST KVM_X86_CPU_FEATURE(0x7, 0, EBX, 1) #define X86_FEATURE_SGX KVM_X86_CPU_FEATURE(0x7, 0, EBX, 2) #define X86_FEATURE_HLE KVM_X86_CPU_FEATURE(0x7, 0, EBX, 4) #define X86_FEATURE_SMEP KVM_X86_CPU_FEATURE(0x7, 0, EBX, 7) #define X86_FEATURE_INVPCID KVM_X86_CPU_FEATURE(0x7, 0, EBX, 10) #define X86_FEATURE_RTM KVM_X86_CPU_FEATURE(0x7, 0, EBX, 11) #define X86_FEATURE_MPX KVM_X86_CPU_FEATURE(0x7, 0, EBX, 14) #define X86_FEATURE_SMAP KVM_X86_CPU_FEATURE(0x7, 0, EBX, 20) #define X86_FEATURE_PCOMMIT KVM_X86_CPU_FEATURE(0x7, 0, EBX, 22) #define X86_FEATURE_CLFLUSHOPT KVM_X86_CPU_FEATURE(0x7, 0, EBX, 23) #define X86_FEATURE_CLWB KVM_X86_CPU_FEATURE(0x7, 0, EBX, 24) #define X86_FEATURE_UMIP KVM_X86_CPU_FEATURE(0x7, 0, ECX, 2) #define X86_FEATURE_PKU KVM_X86_CPU_FEATURE(0x7, 0, ECX, 3) #define X86_FEATURE_OSPKE KVM_X86_CPU_FEATURE(0x7, 0, ECX, 4) #define X86_FEATURE_LA57 KVM_X86_CPU_FEATURE(0x7, 0, ECX, 16) #define X86_FEATURE_RDPID KVM_X86_CPU_FEATURE(0x7, 0, ECX, 22) #define X86_FEATURE_SGX_LC KVM_X86_CPU_FEATURE(0x7, 0, ECX, 30) #define X86_FEATURE_SHSTK KVM_X86_CPU_FEATURE(0x7, 0, ECX, 7) #define X86_FEATURE_IBT KVM_X86_CPU_FEATURE(0x7, 0, EDX, 20) #define X86_FEATURE_AMX_TILE KVM_X86_CPU_FEATURE(0x7, 0, EDX, 24) #define X86_FEATURE_SPEC_CTRL KVM_X86_CPU_FEATURE(0x7, 0, EDX, 26) #define X86_FEATURE_ARCH_CAPABILITIES KVM_X86_CPU_FEATURE(0x7, 0, EDX, 29) #define X86_FEATURE_PKS KVM_X86_CPU_FEATURE(0x7, 0, ECX, 31) #define X86_FEATURE_XTILECFG KVM_X86_CPU_FEATURE(0xD, 0, EAX, 17) #define X86_FEATURE_XTILEDATA KVM_X86_CPU_FEATURE(0xD, 0, EAX, 18) #define X86_FEATURE_XSAVES KVM_X86_CPU_FEATURE(0xD, 1, EAX, 3) #define X86_FEATURE_XFD KVM_X86_CPU_FEATURE(0xD, 1, EAX, 4) #define X86_FEATURE_XTILEDATA_XFD KVM_X86_CPU_FEATURE(0xD, 18, ECX, 2) /* * Extended Leafs, a.k.a. AMD defined */ #define X86_FEATURE_SVM KVM_X86_CPU_FEATURE(0x80000001, 0, ECX, 2) #define X86_FEATURE_PERFCTR_CORE KVM_X86_CPU_FEATURE(0x80000001, 0, ECX, 23) #define X86_FEATURE_PERFCTR_NB KVM_X86_CPU_FEATURE(0x80000001, 0, ECX, 24) #define X86_FEATURE_PERFCTR_LLC KVM_X86_CPU_FEATURE(0x80000001, 0, ECX, 28) #define X86_FEATURE_NX KVM_X86_CPU_FEATURE(0x80000001, 0, EDX, 20) #define X86_FEATURE_FXSR_OPT KVM_X86_CPU_FEATURE(0x80000001, 0, EDX, 25) #define X86_FEATURE_GBPAGES KVM_X86_CPU_FEATURE(0x80000001, 0, EDX, 26) #define X86_FEATURE_RDTSCP KVM_X86_CPU_FEATURE(0x80000001, 0, EDX, 27) #define X86_FEATURE_LM KVM_X86_CPU_FEATURE(0x80000001, 0, EDX, 29) #define X86_FEATURE_INVTSC KVM_X86_CPU_FEATURE(0x80000007, 0, EDX, 8) #define X86_FEATURE_RDPRU KVM_X86_CPU_FEATURE(0x80000008, 0, EBX, 4) #define X86_FEATURE_AMD_IBPB KVM_X86_CPU_FEATURE(0x80000008, 0, EBX, 12) #define X86_FEATURE_NPT KVM_X86_CPU_FEATURE(0x8000000A, 0, EDX, 0) #define X86_FEATURE_LBRV KVM_X86_CPU_FEATURE(0x8000000A, 0, EDX, 1) #define X86_FEATURE_NRIPS KVM_X86_CPU_FEATURE(0x8000000A, 0, EDX, 3) #define X86_FEATURE_TSCRATEMSR KVM_X86_CPU_FEATURE(0x8000000A, 0, EDX, 4) #define X86_FEATURE_PAUSEFILTER KVM_X86_CPU_FEATURE(0x8000000A, 0, EDX, 10) #define X86_FEATURE_PFTHRESHOLD KVM_X86_CPU_FEATURE(0x8000000A, 0, EDX, 12) #define X86_FEATURE_V_VMSAVE_VMLOAD KVM_X86_CPU_FEATURE(0x8000000A, 0, EDX, 15) #define X86_FEATURE_VGIF KVM_X86_CPU_FEATURE(0x8000000A, 0, EDX, 16) #define X86_FEATURE_IDLE_HLT KVM_X86_CPU_FEATURE(0x8000000A, 0, EDX, 30) #define X86_FEATURE_SEV KVM_X86_CPU_FEATURE(0x8000001F, 0, EAX, 1) #define X86_FEATURE_SEV_ES KVM_X86_CPU_FEATURE(0x8000001F, 0, EAX, 3) #define X86_FEATURE_SEV_SNP KVM_X86_CPU_FEATURE(0x8000001F, 0, EAX, 4) #define X86_FEATURE_AUTOIBRS KVM_X86_CPU_FEATURE(0x80000021, 0, EAX, 8) #define X86_FEATURE_GP_ON_USER_CPUID KVM_X86_CPU_FEATURE(0x80000021, 0, EAX, 17) #define X86_FEATURE_PERFMON_V2 KVM_X86_CPU_FEATURE(0x80000022, 0, EAX, 0) #define X86_FEATURE_LBR_PMC_FREEZE KVM_X86_CPU_FEATURE(0x80000022, 0, EAX, 2) /* * KVM defined paravirt features. */ #define X86_FEATURE_KVM_CLOCKSOURCE KVM_X86_CPU_FEATURE(0x40000001, 0, EAX, 0) #define X86_FEATURE_KVM_NOP_IO_DELAY KVM_X86_CPU_FEATURE(0x40000001, 0, EAX, 1) #define X86_FEATURE_KVM_MMU_OP KVM_X86_CPU_FEATURE(0x40000001, 0, EAX, 2) #define X86_FEATURE_KVM_CLOCKSOURCE2 KVM_X86_CPU_FEATURE(0x40000001, 0, EAX, 3) #define X86_FEATURE_KVM_ASYNC_PF KVM_X86_CPU_FEATURE(0x40000001, 0, EAX, 4) #define X86_FEATURE_KVM_STEAL_TIME KVM_X86_CPU_FEATURE(0x40000001, 0, EAX, 5) #define X86_FEATURE_KVM_PV_EOI KVM_X86_CPU_FEATURE(0x40000001, 0, EAX, 6) #define X86_FEATURE_KVM_PV_UNHALT KVM_X86_CPU_FEATURE(0x40000001, 0, EAX, 7) /* Bit 8 apparently isn't used?!?! */ #define X86_FEATURE_KVM_PV_TLB_FLUSH KVM_X86_CPU_FEATURE(0x40000001, 0, EAX, 9) #define X86_FEATURE_KVM_ASYNC_PF_VMEXIT KVM_X86_CPU_FEATURE(0x40000001, 0, EAX, 10) #define X86_FEATURE_KVM_PV_SEND_IPI KVM_X86_CPU_FEATURE(0x40000001, 0, EAX, 11) #define X86_FEATURE_KVM_POLL_CONTROL KVM_X86_CPU_FEATURE(0x40000001, 0, EAX, 12) #define X86_FEATURE_KVM_PV_SCHED_YIELD KVM_X86_CPU_FEATURE(0x40000001, 0, EAX, 13) #define X86_FEATURE_KVM_ASYNC_PF_INT KVM_X86_CPU_FEATURE(0x40000001, 0, EAX, 14) #define X86_FEATURE_KVM_MSI_EXT_DEST_ID KVM_X86_CPU_FEATURE(0x40000001, 0, EAX, 15) #define X86_FEATURE_KVM_HC_MAP_GPA_RANGE KVM_X86_CPU_FEATURE(0x40000001, 0, EAX, 16) #define X86_FEATURE_KVM_MIGRATION_CONTROL KVM_X86_CPU_FEATURE(0x40000001, 0, EAX, 17) /* * Same idea as X86_FEATURE_XXX, but X86_PROPERTY_XXX retrieves a multi-bit * value/property as opposed to a single-bit feature. Again, pack the info * into a 64-bit value to pass by value with no overhead. */ struct kvm_x86_cpu_property { u32 function; u8 index; u8 reg; u8 lo_bit; u8 hi_bit; }; #define KVM_X86_CPU_PROPERTY(fn, idx, gpr, low_bit, high_bit) \ ({ \ struct kvm_x86_cpu_property property = { \ .function = fn, \ .index = idx, \ .reg = KVM_CPUID_##gpr, \ .lo_bit = low_bit, \ .hi_bit = high_bit, \ }; \ \ kvm_static_assert(low_bit < high_bit); \ kvm_static_assert((fn & 0xc0000000) == 0 || \ (fn & 0xc0000000) == 0x40000000 || \ (fn & 0xc0000000) == 0x80000000 || \ (fn & 0xc0000000) == 0xc0000000); \ kvm_static_assert(idx < BIT(sizeof(property.index) * BITS_PER_BYTE)); \ property; \ }) #define X86_PROPERTY_MAX_BASIC_LEAF KVM_X86_CPU_PROPERTY(0, 0, EAX, 0, 31) #define X86_PROPERTY_PMU_VERSION KVM_X86_CPU_PROPERTY(0xa, 0, EAX, 0, 7) #define X86_PROPERTY_PMU_NR_GP_COUNTERS KVM_X86_CPU_PROPERTY(0xa, 0, EAX, 8, 15) #define X86_PROPERTY_PMU_GP_COUNTERS_BIT_WIDTH KVM_X86_CPU_PROPERTY(0xa, 0, EAX, 16, 23) #define X86_PROPERTY_PMU_EBX_BIT_VECTOR_LENGTH KVM_X86_CPU_PROPERTY(0xa, 0, EAX, 24, 31) #define X86_PROPERTY_PMU_EVENTS_MASK KVM_X86_CPU_PROPERTY(0xa, 0, EBX, 0, 12) #define X86_PROPERTY_PMU_FIXED_COUNTERS_BITMASK KVM_X86_CPU_PROPERTY(0xa, 0, ECX, 0, 31) #define X86_PROPERTY_PMU_NR_FIXED_COUNTERS KVM_X86_CPU_PROPERTY(0xa, 0, EDX, 0, 4) #define X86_PROPERTY_PMU_FIXED_COUNTERS_BIT_WIDTH KVM_X86_CPU_PROPERTY(0xa, 0, EDX, 5, 12) #define X86_PROPERTY_SUPPORTED_XCR0_LO KVM_X86_CPU_PROPERTY(0xd, 0, EAX, 0, 31) #define X86_PROPERTY_XSTATE_MAX_SIZE_XCR0 KVM_X86_CPU_PROPERTY(0xd, 0, EBX, 0, 31) #define X86_PROPERTY_XSTATE_MAX_SIZE KVM_X86_CPU_PROPERTY(0xd, 0, ECX, 0, 31) #define X86_PROPERTY_SUPPORTED_XCR0_HI KVM_X86_CPU_PROPERTY(0xd, 0, EDX, 0, 31) #define X86_PROPERTY_XSTATE_TILE_SIZE KVM_X86_CPU_PROPERTY(0xd, 18, EAX, 0, 31) #define X86_PROPERTY_XSTATE_TILE_OFFSET KVM_X86_CPU_PROPERTY(0xd, 18, EBX, 0, 31) #define X86_PROPERTY_AMX_MAX_PALETTE_TABLES KVM_X86_CPU_PROPERTY(0x1d, 0, EAX, 0, 31) #define X86_PROPERTY_AMX_TOTAL_TILE_BYTES KVM_X86_CPU_PROPERTY(0x1d, 1, EAX, 0, 15) #define X86_PROPERTY_AMX_BYTES_PER_TILE KVM_X86_CPU_PROPERTY(0x1d, 1, EAX, 16, 31) #define X86_PROPERTY_AMX_BYTES_PER_ROW KVM_X86_CPU_PROPERTY(0x1d, 1, EBX, 0, 15) #define X86_PROPERTY_AMX_NR_TILE_REGS KVM_X86_CPU_PROPERTY(0x1d, 1, EBX, 16, 31) #define X86_PROPERTY_AMX_MAX_ROWS KVM_X86_CPU_PROPERTY(0x1d, 1, ECX, 0, 15) #define X86_PROPERTY_MAX_KVM_LEAF KVM_X86_CPU_PROPERTY(0x40000000, 0, EAX, 0, 31) #define X86_PROPERTY_MAX_EXT_LEAF KVM_X86_CPU_PROPERTY(0x80000000, 0, EAX, 0, 31) #define X86_PROPERTY_MAX_PHY_ADDR KVM_X86_CPU_PROPERTY(0x80000008, 0, EAX, 0, 7) #define X86_PROPERTY_MAX_VIRT_ADDR KVM_X86_CPU_PROPERTY(0x80000008, 0, EAX, 8, 15) #define X86_PROPERTY_GUEST_MAX_PHY_ADDR KVM_X86_CPU_PROPERTY(0x80000008, 0, EAX, 16, 23) #define X86_PROPERTY_SEV_C_BIT KVM_X86_CPU_PROPERTY(0x8000001F, 0, EBX, 0, 5) #define X86_PROPERTY_PHYS_ADDR_REDUCTION KVM_X86_CPU_PROPERTY(0x8000001F, 0, EBX, 6, 11) #define X86_PROPERTY_NR_PERFCTR_CORE KVM_X86_CPU_PROPERTY(0x80000022, 0, EBX, 0, 3) #define X86_PROPERTY_NR_PERFCTR_NB KVM_X86_CPU_PROPERTY(0x80000022, 0, EBX, 10, 15) #define X86_PROPERTY_MAX_CENTAUR_LEAF KVM_X86_CPU_PROPERTY(0xC0000000, 0, EAX, 0, 31) /* * Intel's architectural PMU events are bizarre. They have a "feature" bit * that indicates the feature is _not_ supported, and a property that states * the length of the bit mask of unsupported features. A feature is supported * if the size of the bit mask is larger than the "unavailable" bit, and said * bit is not set. Fixed counters also bizarre enumeration, but inverted from * arch events for general purpose counters. Fixed counters are supported if a * feature flag is set **OR** the total number of fixed counters is greater * than index of the counter. * * Wrap the events for general purpose and fixed counters to simplify checking * whether or not a given architectural event is supported. */ struct kvm_x86_pmu_feature { struct kvm_x86_cpu_feature f; }; #define KVM_X86_PMU_FEATURE(__reg, __bit) \ ({ \ struct kvm_x86_pmu_feature feature = { \ .f = KVM_X86_CPU_FEATURE(0xa, 0, __reg, __bit), \ }; \ \ kvm_static_assert(KVM_CPUID_##__reg == KVM_CPUID_EBX || \ KVM_CPUID_##__reg == KVM_CPUID_ECX); \ feature; \ }) #define X86_PMU_FEATURE_CPU_CYCLES KVM_X86_PMU_FEATURE(EBX, 0) #define X86_PMU_FEATURE_INSNS_RETIRED KVM_X86_PMU_FEATURE(EBX, 1) #define X86_PMU_FEATURE_REFERENCE_CYCLES KVM_X86_PMU_FEATURE(EBX, 2) #define X86_PMU_FEATURE_LLC_REFERENCES KVM_X86_PMU_FEATURE(EBX, 3) #define X86_PMU_FEATURE_LLC_MISSES KVM_X86_PMU_FEATURE(EBX, 4) #define X86_PMU_FEATURE_BRANCH_INSNS_RETIRED KVM_X86_PMU_FEATURE(EBX, 5) #define X86_PMU_FEATURE_BRANCHES_MISPREDICTED KVM_X86_PMU_FEATURE(EBX, 6) #define X86_PMU_FEATURE_TOPDOWN_SLOTS KVM_X86_PMU_FEATURE(EBX, 7) #define X86_PMU_FEATURE_TOPDOWN_BE_BOUND KVM_X86_PMU_FEATURE(EBX, 8) #define X86_PMU_FEATURE_TOPDOWN_BAD_SPEC KVM_X86_PMU_FEATURE(EBX, 9) #define X86_PMU_FEATURE_TOPDOWN_FE_BOUND KVM_X86_PMU_FEATURE(EBX, 10) #define X86_PMU_FEATURE_TOPDOWN_RETIRING KVM_X86_PMU_FEATURE(EBX, 11) #define X86_PMU_FEATURE_LBR_INSERTS KVM_X86_PMU_FEATURE(EBX, 12) #define X86_PMU_FEATURE_INSNS_RETIRED_FIXED KVM_X86_PMU_FEATURE(ECX, 0) #define X86_PMU_FEATURE_CPU_CYCLES_FIXED KVM_X86_PMU_FEATURE(ECX, 1) #define X86_PMU_FEATURE_REFERENCE_TSC_CYCLES_FIXED KVM_X86_PMU_FEATURE(ECX, 2) #define X86_PMU_FEATURE_TOPDOWN_SLOTS_FIXED KVM_X86_PMU_FEATURE(ECX, 3) static inline unsigned int x86_family(unsigned int eax) { unsigned int x86; x86 = (eax >> 8) & 0xf; if (x86 == 0xf) x86 += (eax >> 20) & 0xff; return x86; } static inline unsigned int x86_model(unsigned int eax) { return ((eax >> 12) & 0xf0) | ((eax >> 4) & 0x0f); } #define PHYSICAL_PAGE_MASK GENMASK_ULL(51, 12) #define PAGE_SHIFT 12 #define PAGE_SIZE (1ULL << PAGE_SHIFT) #define PAGE_MASK (~(PAGE_SIZE-1) & PHYSICAL_PAGE_MASK) #define HUGEPAGE_SHIFT(x) (PAGE_SHIFT + (((x) - 1) * 9)) #define HUGEPAGE_SIZE(x) (1UL << HUGEPAGE_SHIFT(x)) #define HUGEPAGE_MASK(x) (~(HUGEPAGE_SIZE(x) - 1) & PHYSICAL_PAGE_MASK) #define PTE_GET_PA(pte) ((pte) & PHYSICAL_PAGE_MASK) #define PTE_GET_PFN(pte) (PTE_GET_PA(pte) >> PAGE_SHIFT) struct guest_regs { u64 rax; u64 rcx; u64 rdx; u64 rbx; u64 rsp; u64 rbp; u64 rsi; u64 rdi; u64 r8; u64 r9; u64 r10; u64 r11; u64 r12; u64 r13; u64 r14; u64 r15; u64 rflags; }; extern struct guest_regs guest_regs; #define GUEST_REG_OFFSET(name) \ [off_##name] "i" (offsetof(struct guest_regs, name)) #define GUEST_REGS_OFFSETS \ GUEST_REG_OFFSET(rax), \ GUEST_REG_OFFSET(rcx), \ GUEST_REG_OFFSET(rdx), \ GUEST_REG_OFFSET(rbx), \ GUEST_REG_OFFSET(rsp), \ GUEST_REG_OFFSET(rbp), \ GUEST_REG_OFFSET(rsi), \ GUEST_REG_OFFSET(rdi), \ GUEST_REG_OFFSET(r8), \ GUEST_REG_OFFSET(r9), \ GUEST_REG_OFFSET(r10), \ GUEST_REG_OFFSET(r11), \ GUEST_REG_OFFSET(r12), \ GUEST_REG_OFFSET(r13), \ GUEST_REG_OFFSET(r14), \ GUEST_REG_OFFSET(r15), \ GUEST_REG_OFFSET(rflags) #define GUEST_REG(name) "guest_regs + %c[off_" #name "]" #define GUEST_SWITCH_GPR_ASM(name) \ "xchg %%" #name ", " GUEST_REG(name) "\n\t" struct desc64 { u16 limit0; u16 base0; unsigned base1:8, type:4, s:1, dpl:2, p:1; unsigned limit1:4, avl:1, l:1, db:1, g:1, base2:8; u32 base3; u32 zero1; } __attribute__((packed)); struct desc_ptr { u16 size; u64 address; } __attribute__((packed)); struct kvm_x86_state { struct kvm_xsave *xsave; struct kvm_vcpu_events events; struct kvm_mp_state mp_state; struct kvm_regs regs; struct kvm_xcrs xcrs; struct kvm_sregs sregs; struct kvm_debugregs debugregs; union { struct kvm_nested_state nested; char nested_[16384]; }; struct kvm_msrs msrs; }; static inline u64 get_desc64_base(const struct desc64 *desc) { return (u64)desc->base3 << 32 | (u64)desc->base2 << 24 | (u64)desc->base1 << 16 | (u64)desc->base0; } static inline u64 rdtsc(void) { u32 eax, edx; u64 tsc_val; /* * The lfence is to wait (on Intel CPUs) until all previous * instructions have been executed. If software requires RDTSC to be * executed prior to execution of any subsequent instruction, it can * execute LFENCE immediately after RDTSC */ __asm__ __volatile__("lfence; rdtsc; lfence" : "=a"(eax), "=d"(edx)); tsc_val = ((u64)edx) << 32 | eax; return tsc_val;