// SPDX-License-Identifier: GPL-2.0-only/* * Copyright (C) 2018, Google LLC. */ #include "linux/bitmap.h"#include "test_util.h"#include "kvm_util.h"#include "pmu.h"#include "processor.h"#include "smm.h"#include "svm_util.h"#include "sev.h"#include "vmx.h" #ifndef NUM_INTERRUPTS#define NUM_INTERRUPTS 256#endif #define KERNEL_CS	0x8#define KERNEL_DS	0x10#define KERNEL_TSS	0x18 gva_t exception_handlers;bool host_cpu_is_amd;bool host_cpu_is_intel;bool host_cpu_is_hygon;bool host_cpu_is_amd_compatible;bool is_forced_emulation_enabled;u64 guest_tsc_khz;struct kvm_mmu guest_mmu; struct guest_regs guest_regs; const char *ex_str(int vector){	switch (vector) {#define VEC_STR(v) case v##_VECTOR: return "#" #v	case DE_VECTOR: return "no exception";	case KVM_MAGIC_DE_VECTOR: return "#DE";	VEC_STR(DB);	VEC_STR(NMI);	VEC_STR(BP);	VEC_STR(OF);	VEC_STR(BR);	VEC_STR(UD);	VEC_STR(NM);	VEC_STR(DF);	VEC_STR(TS);	VEC_STR(NP);	VEC_STR(SS);	VEC_STR(GP);	VEC_STR(PF);	VEC_STR(MF);	VEC_STR(AC);	VEC_STR(MC);	VEC_STR(XM);	VEC_STR(VE);	VEC_STR(CP);	VEC_STR(HV);	VEC_STR(VC);	VEC_STR(SX);	default: return "#??";#undef VEC_STR	}} static void regs_dump(FILE *stream, struct kvm_regs *regs, u8 indent){	fprintf(stream, "%*srax: 0x%.16llx rbx: 0x%.16llx "		"rcx: 0x%.16llx rdx: 0x%.16llx\n",		indent, "",		regs->rax, regs->rbx, regs->rcx, regs->rdx);	fprintf(stream, "%*srsi: 0x%.16llx rdi: 0x%.16llx "		"rsp: 0x%.16llx rbp: 0x%.16llx\n",		indent, "",		regs->rsi, regs->rdi, regs->rsp, regs->rbp);	fprintf(stream, "%*sr8:  0x%.16llx r9:  0x%.16llx "		"r10: 0x%.16llx r11: 0x%.16llx\n",		indent, "",		regs->r8, regs->r9, regs->r10, regs->r11);	fprintf(stream, "%*sr12: 0x%.16llx r13: 0x%.16llx "		"r14: 0x%.16llx r15: 0x%.16llx\n",		indent, "",		regs->r12, regs->r13, regs->r14, regs->r15);	fprintf(stream, "%*srip: 0x%.16llx rfl: 0x%.16llx\n",		indent, "",		regs->rip, regs->rflags);} static void segment_dump(FILE *stream, struct kvm_segment *segment,			 u8 indent){	fprintf(stream, "%*sbase: 0x%.16llx limit: 0x%.8x "		"selector: 0x%.4x type: 0x%.2x\n",		indent, "", segment->base, segment->limit,		segment->selector, segment->type);	fprintf(stream, "%*spresent: 0x%.2x dpl: 0x%.2x "		"db: 0x%.2x s: 0x%.2x l: 0x%.2x\n",		indent, "", segment->present, segment->dpl,		segment->db, segment->s, segment->l);	fprintf(stream, "%*sg: 0x%.2x avl: 0x%.2x "		"unusable: 0x%.2x padding: 0x%.2x\n",		indent, "", segment->g, segment->avl,		segment->unusable, segment->padding);} static void dtable_dump(FILE *stream, struct kvm_dtable *dtable,			u8 indent){	fprintf(stream, "%*sbase: 0x%.16llx limit: 0x%.4x "		"padding: 0x%.4x 0x%.4x 0x%.4x\n",		indent, "", dtable->base, dtable->limit,		dtable->padding[0], dtable->padding[1], dtable->padding[2]);} static void sregs_dump(FILE *stream, struct kvm_sregs *sregs, u8 indent){	unsigned int i; 	fprintf(stream, "%*scs:\n", indent, "");	segment_dump(stream, &sregs->cs, indent + 2);	fprintf(stream, "%*sds:\n", indent, "");	segment_dump(stream, &sregs->ds, indent + 2);	fprintf(stream, "%*ses:\n", indent, "");	segment_dump(stream, &sregs->es, indent + 2);	fprintf(stream, "%*sfs:\n", indent, "");	segment_dump(stream, &sregs->fs, indent + 2);	fprintf(stream, "%*sgs:\n", indent, "");	segment_dump(stream, &sregs->gs, indent + 2);	fprintf(stream, "%*sss:\n", indent, "");	segment_dump(stream, &sregs->ss, indent + 2);	fprintf(stream, "%*str:\n", indent, "");	segment_dump(stream, &sregs->tr, indent + 2);	fprintf(stream, "%*sldt:\n", indent, "");	segment_dump(stream, &sregs->ldt, indent + 2); 	fprintf(stream, "%*sgdt:\n", indent, "");	dtable_dump(stream, &sregs->gdt, indent + 2);	fprintf(stream, "%*sidt:\n", indent, "");	dtable_dump(stream, &sregs->idt, indent + 2); 	fprintf(stream, "%*scr0: 0x%.16llx cr2: 0x%.16llx "		"cr3: 0x%.16llx cr4: 0x%.16llx\n",		indent, "",		sregs->cr0, sregs->cr2, sregs->cr3, sregs->cr4);	fprintf(stream, "%*scr8: 0x%.16llx efer: 0x%.16llx "		"apic_base: 0x%.16llx\n",		indent, "",		sregs->cr8, sregs->efer, sregs->apic_base); 	fprintf(stream, "%*sinterrupt_bitmap:\n", indent, "");	for (i = 0; i < (KVM_NR_INTERRUPTS + 63) / 64; i++) {		fprintf(stream, "%*s%.16llx\n", indent + 2, "",			sregs->interrupt_bitmap[i]);	}} bool kvm_is_tdp_enabled(void){	if (host_cpu_is_intel)		return get_kvm_intel_param_bool("ept");	else		return get_kvm_amd_param_bool("npt");} static void virt_mmu_init(struct kvm_vm *vm, struct kvm_mmu *mmu,			  struct pte_masks *pte_masks){	/* If needed, create the top-level page table. */	if (!mmu->pgd_created) {		mmu->pgd = vm_alloc_page_table(vm);		mmu->pgd_created = true;		mmu->arch.pte_masks = *pte_masks;	} 	TEST_ASSERT(mmu->pgtable_levels == 4 || mmu->pgtable_levels == 5,		    "Selftests MMU only supports 4-level and 5-level paging, not %u-level paging",		    mmu->pgtable_levels);} void virt_arch_pgd_alloc(struct kvm_vm *vm){	TEST_ASSERT(vm->mode == VM_MODE_PXXVYY_4K,		    "Unknown or unsupported guest mode: 0x%x", vm->mode); 	struct pte_masks pte_masks = (struct pte_masks){		.present	=	BIT_ULL(0),		.writable	=	BIT_ULL(1),		.user		=	BIT_ULL(2),		.accessed	=	BIT_ULL(5),		.dirty		=	BIT_ULL(6),		.huge		=	BIT_ULL(7),		.nx		=	BIT_ULL(63),		.executable	=	0,		.c		=	vm->arch.c_bit,		.s		=	vm->arch.s_bit,	}; 	virt_mmu_init(vm, &vm->mmu, &pte_masks);} void tdp_mmu_init(struct kvm_vm *vm, int pgtable_levels,		  struct pte_masks *pte_masks){	TEST_ASSERT(!vm->stage2_mmu.pgtable_levels, "TDP MMU already initialized"); 	vm->stage2_mmu.pgtable_levels = pgtable_levels;	virt_mmu_init(vm, &vm->stage2_mmu, pte_masks);} static void *virt_get_pte(struct kvm_vm *vm, struct kvm_mmu *mmu,			  u64 *parent_pte, gva_t gva, int level){	u64 pt_gpa = PTE_GET_PA(*parent_pte);	u64 *page_table = addr_gpa2hva(vm, pt_gpa);	int index = (gva >> PG_LEVEL_SHIFT(level)) & 0x1ffu; 	TEST_ASSERT((*parent_pte == mmu->pgd) || is_present_pte(mmu, parent_pte),		    "Parent PTE (level %d) not PRESENT for gva: 0x%08lx",		    level + 1, gva); 	return &page_table[index];} static u64 *virt_create_upper_pte(struct kvm_vm *vm,				  struct kvm_mmu *mmu,				  u64 *parent_pte,				  gva_t gva,				  gpa_t gpa,				  int current_level,				  int target_level){	u64 *pte = virt_get_pte(vm, mmu, parent_pte, gva, current_level); 	gpa = vm_untag_gpa(vm, gpa); 	if (!is_present_pte(mmu, pte)) {		*pte = PTE_PRESENT_MASK(mmu) | PTE_READABLE_MASK(mmu) |		       PTE_WRITABLE_MASK(mmu) | PTE_EXECUTABLE_MASK(mmu) |		       PTE_ALWAYS_SET_MASK(mmu);		if (current_level == target_level)			*pte |= PTE_HUGE_MASK(mmu) | (gpa & PHYSICAL_PAGE_MASK);		else			*pte |= vm_alloc_page_table(vm) & PHYSICAL_PAGE_MASK;	} else {		/*		 * Entry already present.  Assert that the caller doesn't want		 * a hugepage at this level, and that there isn't a hugepage at		 * this level.		 */		TEST_ASSERT(current_level != target_level,			    "Cannot create hugepage at level: %u, gva: 0x%lx",			    current_level, gva);		TEST_ASSERT(!is_huge_pte(mmu, pte),			    "Cannot create page table at level: %u, gva: 0x%lx",			    current_level, gva);	}	return pte;} void __virt_pg_map(struct kvm_vm *vm, struct kvm_mmu *mmu, gva_t gva,		   gpa_t gpa, int level){	const u64 pg_size = PG_LEVEL_SIZE(level);	u64 *pte = &mmu->pgd;	int current_level; 	TEST_ASSERT(vm->mode == VM_MODE_PXXVYY_4K,		    "Unknown or unsupported guest mode: 0x%x", vm->mode); 	TEST_ASSERT((gva % pg_size) == 0,		    "Virtual address not aligned,\n"		    "gva: 0x%lx page size: 0x%lx", gva, pg_size);	TEST_ASSERT(sparsebit_is_set(vm->vpages_valid, (gva >> vm->page_shift)),		    "Invalid virtual address, gva: 0x%lx", gva);	TEST_ASSERT((gpa % pg_size) == 0,		    "Physical address not aligned,\n"		    "  gpa: 0x%lx page size: 0x%lx", gpa, pg_size);	TEST_ASSERT((gpa >> vm->page_shift) <= vm->max_gfn,		    "Physical address beyond maximum supported,\n"		    "  gpa: 0x%lx vm->max_gfn: 0x%lx vm->page_size: 0x%x",		    gpa, vm->max_gfn, vm->page_size);	TEST_ASSERT(vm_untag_gpa(vm, gpa) == gpa,		    "Unexpected bits in gpa: %lx", gpa); 	TEST_ASSERT(!PTE_EXECUTABLE_MASK(mmu) || !PTE_NX_MASK(mmu),		    "X and NX bit masks cannot be used simultaneously"); 	/*	 * Allocate upper level page tables, if not already present.  Return	 * early if a hugepage was created.	 */	for (current_level = mmu->pgtable_levels;	     current_level > PG_LEVEL_4K;	     current_level--) {		pte = virt_create_upper_pte(vm, mmu, pte, gva, gpa,					    current_level, level);		if (is_huge_pte(mmu, pte))			return;	} 	/* Fill in page table entry. */	pte = virt_get_pte(vm, mmu, pte, gva, PG_LEVEL_4K);	TEST_ASSERT(!is_present_pte(mmu, pte),		    "PTE already present for 4k page at gva: 0x%lx", gva);	*pte = PTE_PRESENT_MASK(mmu) | PTE_READABLE_MASK(mmu) |	       PTE_WRITABLE_MASK(mmu) | PTE_EXECUTABLE_MASK(mmu) |	       PTE_ALWAYS_SET_MASK(mmu) | (gpa & PHYSICAL_PAGE_MASK); 	/*	 * Neither SEV nor TDX supports shared page tables, so only the final	 * leaf PTE needs manually set the C/S-bit.	 */	if (vm_is_gpa_protected(vm, gpa))		*pte |= PTE_C_BIT_MASK(mmu);	else		*pte |= PTE_S_BIT_MASK(mmu);} void virt_arch_pg_map(struct kvm_vm *vm, gva_t gva, gpa_t gpa){	__virt_pg_map(vm, &vm->mmu, gva, gpa, PG_LEVEL_4K);} void virt_map_level(struct kvm_vm *vm, gva_t gva, gpa_t gpa,		    u64 nr_bytes, int level){	u64 pg_size = PG_LEVEL_SIZE(level);	u64 nr_pages = nr_bytes / pg_size;	int i; 	TEST_ASSERT(nr_bytes % pg_size == 0,		    "Region size not aligned: nr_bytes: 0x%lx, page size: 0x%lx",		    nr_bytes, pg_size); 	for (i = 0; i < nr_pages; i++) {		__virt_pg_map(vm, &vm->mmu, gva, gpa, level);		sparsebit_set_num(vm->vpages_mapped, gva >> vm->page_shift,				  nr_bytes / PAGE_SIZE); 		gva += pg_size;		gpa += pg_size;	}} static bool vm_is_target_pte(struct kvm_mmu *mmu, u64 *pte,			     int *level, int current_level){	if (is_huge_pte(mmu, pte)) {		TEST_ASSERT(*level == PG_LEVEL_NONE ||			    *level == current_level,			    "Unexpected hugepage at level %d", current_level);		*level = current_level;	} 	return *level == current_level;} static u64 *__vm_get_page_table_entry(struct kvm_vm *vm,				      struct kvm_mmu *mmu,				      gva_t gva,				      int *level){	int va_width = 12 + (mmu->pgtable_levels) * 9;	u64 *pte = &mmu->pgd;	int current_level; 	TEST_ASSERT(!vm->arch.is_pt_protected,		    "Walking page tables of protected guests is impossible"); 	TEST_ASSERT(*level >= PG_LEVEL_NONE && *level <= mmu->pgtable_levels,		    "Invalid PG_LEVEL_* '%d'", *level); 	TEST_ASSERT(vm->mode == VM_MODE_PXXVYY_4K,		    "Unknown or unsupported guest mode: 0x%x", vm->mode);	TEST_ASSERT(sparsebit_is_set(vm->vpages_valid, (gva >> vm->page_shift)),		    "Invalid virtual address, gva: 0x%lx", gva);	/*	 * Check that the gva is a sign-extended va_width value.	 */	TEST_ASSERT(gva == (((s64)gva << (64 - va_width) >> (64 - va_width))),		    "Canonical check failed.  The virtual address is invalid."); 	for (current_level = mmu->pgtable_levels;	     current_level > PG_LEVEL_4K;	     current_level--) {		pte = virt_get_pte(vm, mmu, pte, gva, current_level);		if (vm_is_target_pte(mmu, pte, level, current_level))			return pte;	} 	return virt_get_pte(vm, mmu, pte, gva, PG_LEVEL_4K);} u64 *tdp_get_pte(struct kvm_vm *vm, u64 l2_gpa){	int level = PG_LEVEL_4K; 	return __vm_get_page_table_entry(vm, &vm->stage2_mmu, l2_gpa, &level);} u64 *vm_get_pte(struct kvm_vm *vm, gva_t gva){	int level = PG_LEVEL_4K; 	return __vm_get_page_table_entry(vm, &vm->mmu, gva, &level);} void virt_arch_dump(FILE *stream, struct kvm_vm *vm, u8 indent){	struct kvm_mmu *mmu = &vm->mmu;	u64 *pml4e, *pml4e_start;	u64 *pdpe, *pdpe_start;	u64 *pde, *pde_start;	u64 *pte, *pte_start; 	if (!mmu->pgd_created)		return; 	fprintf(stream, "%*s                                          "		"                no\n", indent, "");	fprintf(stream, "%*s      index hvaddr         gpaddr         "		"addr         w exec dirty\n",		indent, "");	pml4e_start = (u64 *)addr_gpa2hva(vm, mmu->pgd);	for (u16 n1 = 0; n1 <= 0x1ffu; n1++) {		pml4e = &pml4e_start[n1];		if (!is_present_pte(mmu, pml4e))			continue;		fprintf(stream, "%*spml4e 0x%-3zx %p 0x%-12lx 0x%-10llx %u "			" %u\n",			indent, "",			pml4e - pml4e_start, pml4e,			addr_hva2gpa(vm, pml4e), PTE_GET_PFN(*pml4e),			is_writable_pte(mmu, pml4e), is_nx_pte(mmu, pml4e)); 		pdpe_start = addr_gpa2hva(vm, *pml4e & PHYSICAL_PAGE_MASK);		for (u16 n2 = 0; n2 <= 0x1ffu; n2++) {			pdpe = &pdpe_start[n2];			if (!is_present_pte(mmu, pdpe))				continue;			fprintf(stream, "%*spdpe  0x%-3zx %p 0x%-12lx 0x%-10llx "				"%u  %u\n",				indent, "",				pdpe - pdpe_start, pdpe,				addr_hva2gpa(vm, pdpe),				PTE_GET_PFN(*pdpe), is_writable_pte(mmu, pdpe),				is_nx_pte(mmu, pdpe)); 			pde_start = addr_gpa2hva(vm, *pdpe & PHYSICAL_PAGE_MASK);			for (u16 n3 = 0; n3 <= 0x1ffu; n3++) {				pde = &pde_start[n3];				if (!is_present_pte(mmu, pde))					continue;				fprintf(stream, "%*spde   0x%-3zx %p "					"0x%-12lx 0x%-10llx %u  %u\n",					indent, "", pde - pde_start, pde,					addr_hva2gpa(vm, pde),					PTE_GET_PFN(*pde), is_writable_pte(mmu, pde),					is_nx_pte(mmu, pde)); 				pte_start = addr_gpa2hva(vm, *pde & PHYSICAL_PAGE_MASK);				for (u16 n4 = 0; n4 <= 0x1ffu; n4++) {					pte = &pte_start[n4];					if (!is_present_pte(mmu, pte))						continue;					fprintf(stream, "%*spte   0x%-3zx %p "						"0x%-12lx 0x%-10llx %u  %u "						"    %u    0x%-10lx\n",						indent, "",						pte - pte_start, pte,						addr_hva2gpa(vm, pte),						PTE_GET_PFN(*pte),						is_writable_pte(mmu, pte),						is_nx_pte(mmu, pte),						is_dirty_pte(mmu, pte),						((u64)n1 << 27)							| ((u64)n2 << 18)							| ((u64)n3 << 9)							| ((u64)n4));				}			}		}	}} void vm_enable_tdp(struct kvm_vm *vm){	if (kvm_cpu_has(X86_FEATURE_VMX))		vm_enable_ept(vm);	else		vm_enable_npt(vm);} bool kvm_cpu_has_tdp(void){	return kvm_cpu_has_ept() || kvm_cpu_has_npt();} 

Lines 1–500 of 1490. Load more