// SPDX-License-Identifier: GPL-2.0-only/* * Copyright (C) 2017 - Columbia University and Linaro Ltd. * Author: Jintack Lim <jintack.lim@linaro.org> */ #include <linux/bitfield.h>#include <linux/kvm.h>#include <linux/kvm_host.h> #include <asm/fixmap.h>#include <asm/kvm_arm.h>#include <asm/kvm_emulate.h>#include <asm/kvm_mmu.h>#include <asm/kvm_nested.h>#include <asm/sysreg.h> #include "sys_regs.h"#include "vgic/vgic.h" struct vncr_tlb {	/* The guest's VNCR_EL2 */	u64			gva;	struct s1_walk_info	wi;	struct s1_walk_result	wr; 	u64			hpa;	bool			hpa_writable; 	/* -1 when not mapped on a CPU */	atomic_t		cpu; 	/*	 * true if the TLB is valid. Can only be changed with the	 * mmu_lock held.	 */	bool			valid;}; /* * Ratio of live shadow S2 MMU per vcpu. This is a trade-off between * memory usage and potential number of different sets of S2 PTs in * the guests. Running out of S2 MMUs only affects performance (we * will invalidate them more often). */#define S2_MMU_PER_VCPU		2 void kvm_init_nested(struct kvm *kvm){	kvm->arch.nested_mmus = NULL;	kvm->arch.nested_mmus_size = 0;	atomic_set(&kvm->arch.vncr_tlb_count, 0);} static int init_nested_s2_mmu(struct kvm *kvm, struct kvm_s2_mmu *mmu){	/*	 * We only initialise the IPA range on the canonical MMU, which	 * defines the contract between KVM and userspace on where the	 * "hardware" is in the IPA space. This affects the validity of MMIO	 * exits forwarded to userspace, for example.	 *	 * For nested S2s, we use the PARange as exposed to the guest, as it	 * is allowed to use it at will to expose whatever memory map it	 * wants to its own guests as it would be on real HW.	 */	return kvm_init_stage2_mmu(kvm, mmu, kvm_get_pa_bits(kvm));} int kvm_vcpu_init_nested(struct kvm_vcpu *vcpu){	struct kvm *kvm = vcpu->kvm;	struct kvm_s2_mmu *tmp;	int num_mmus, ret = 0; 	if (test_bit(KVM_ARM_VCPU_HAS_EL2_E2H0, kvm->arch.vcpu_features) &&	    !cpus_have_final_cap(ARM64_HAS_HCR_NV1))		return -EINVAL; 	if (!vcpu->arch.ctxt.vncr_array)		vcpu->arch.ctxt.vncr_array = (u64 *)__get_free_page(GFP_KERNEL_ACCOUNT |								    __GFP_ZERO); 	if (!vcpu->arch.ctxt.vncr_array)		return -ENOMEM; 	/*	 * Let's treat memory allocation failures as benign: If we fail to	 * allocate anything, return an error and keep the allocated array	 * alive. Userspace may try to recover by initializing the vcpu	 * again, and there is no reason to affect the whole VM for this.	 */	num_mmus = atomic_read(&kvm->online_vcpus) * S2_MMU_PER_VCPU; 	if (num_mmus > kvm->arch.nested_mmus_size) {		tmp = kvzalloc_objs(*tmp, num_mmus, GFP_KERNEL_ACCOUNT);		if (!tmp)			return -ENOMEM; 		write_lock(&kvm->mmu_lock); 		if (kvm->arch.nested_mmus_size) {			memcpy(tmp, kvm->arch.nested_mmus,			       size_mul(sizeof(*tmp), kvm->arch.nested_mmus_size)); 			for (int i = 0; i < kvm->arch.nested_mmus_size; i++)				tmp[i].pgt->mmu = &tmp[i];		} 		swap(kvm->arch.nested_mmus, tmp); 		write_unlock(&kvm->mmu_lock); 		kvfree(tmp);	} 	for (int i = kvm->arch.nested_mmus_size; !ret && i < num_mmus; i++)		ret = init_nested_s2_mmu(kvm, &kvm->arch.nested_mmus[i]); 	if (ret) {		for (int i = kvm->arch.nested_mmus_size; i < num_mmus; i++)			kvm_free_stage2_pgd(&kvm->arch.nested_mmus[i]); 		free_page((unsigned long)vcpu->arch.ctxt.vncr_array);		vcpu->arch.ctxt.vncr_array = NULL; 		return ret;	} 	kvm->arch.nested_mmus_size = num_mmus; 	return 0;} struct s2_walk_info {	u64		baddr;	unsigned int	max_oa_bits;	unsigned int	pgshift;	unsigned int	sl;	unsigned int	t0sz;	bool		be;	bool		ha;}; static u32 compute_fsc(int level, u32 fsc){	return fsc | (level & 0x3);} static int esr_s2_fault(struct kvm_vcpu *vcpu, int level, u32 fsc){	u32 esr; 	esr = kvm_vcpu_get_esr(vcpu) & ~ESR_ELx_FSC;	esr |= compute_fsc(level, fsc);	return esr;} static int get_ia_size(struct s2_walk_info *wi){	return 64 - wi->t0sz;} static int check_base_s2_limits(struct kvm_vcpu *vcpu, struct s2_walk_info *wi,				int level, int input_size, int stride){	int start_size, pa_max; 	pa_max = kvm_get_pa_bits(vcpu->kvm); 	/* Check translation limits */	switch (BIT(wi->pgshift)) {	case SZ_64K:		if (level == 0 || (level == 1 && pa_max <= 42))			return -EFAULT;		break;	case SZ_16K:		if (level == 0 || (level == 1 && pa_max <= 40))			return -EFAULT;		break;	case SZ_4K:		if (level < 0 || (level == 0 && pa_max <= 42))			return -EFAULT;		break;	} 	/* Check input size limits */	if (input_size > pa_max)		return -EFAULT; 	/* Check number of entries in starting level table */	start_size = input_size - ((3 - level) * stride + wi->pgshift);	if (start_size < 1 || start_size > stride + 4)		return -EFAULT; 	return 0;} /* Check if output is within boundaries */static int check_output_size(struct s2_walk_info *wi, phys_addr_t output){	unsigned int output_size = wi->max_oa_bits; 	if (output_size != 48 && (output & GENMASK_ULL(47, output_size)))		return -1; 	return 0;} static int read_guest_s2_desc(struct kvm_vcpu *vcpu, phys_addr_t pa, u64 *desc,			      struct s2_walk_info *wi){	u64 val;	int r; 	r = kvm_read_guest(vcpu->kvm, pa, &val, sizeof(val));	if (r)		return r; 	/*	 * Handle reversedescriptors if endianness differs between the	 * host and the guest hypervisor.	 */	if (wi->be)		*desc = be64_to_cpu((__force __be64)val);	else		*desc = le64_to_cpu((__force __le64)val); 	return 0;} static int swap_guest_s2_desc(struct kvm_vcpu *vcpu, phys_addr_t pa, u64 old, u64 new,			      struct s2_walk_info *wi){	if (wi->be) {		old = (__force u64)cpu_to_be64(old);		new = (__force u64)cpu_to_be64(new);	} else {		old = (__force u64)cpu_to_le64(old);		new = (__force u64)cpu_to_le64(new);	} 	return __kvm_at_swap_desc(vcpu->kvm, pa, old, new);} /* * This is essentially a C-version of the pseudo code from the ARM ARM * AArch64.TranslationTableWalk  function.  I strongly recommend looking at * that pseudocode in trying to understand this. * * Must be called with the kvm->srcu read lock held */static int walk_nested_s2_pgd(struct kvm_vcpu *vcpu, phys_addr_t ipa,			      struct s2_walk_info *wi, struct kvm_s2_trans *out){	int first_block_level, level, stride, input_size, base_lower_bound;	phys_addr_t base_addr;	unsigned int addr_top, addr_bottom;	u64 desc, new_desc;  /* page table entry */	int ret;	phys_addr_t paddr; 	switch (BIT(wi->pgshift)) {	default:	case SZ_64K:	case SZ_16K:		level = 3 - wi->sl;		first_block_level = 2;		break;	case SZ_4K:		level = 2 - wi->sl;		first_block_level = 1;		break;	} 	stride = wi->pgshift - 3;	input_size = get_ia_size(wi);	if (input_size > 48 || input_size < 25)		return -EFAULT; 	ret = check_base_s2_limits(vcpu, wi, level, input_size, stride);	if (WARN_ON(ret)) {		out->esr = compute_fsc(0, ESR_ELx_FSC_FAULT);		return ret;	} 	base_lower_bound = 3 + input_size - ((3 - level) * stride +			   wi->pgshift);	base_addr = wi->baddr & GENMASK_ULL(47, base_lower_bound); 	if (check_output_size(wi, base_addr)) {		/* R_BFHQH */		out->esr = compute_fsc(0, ESR_ELx_FSC_ADDRSZ);		return 1;	} 	addr_top = input_size - 1; 	while (1) {		phys_addr_t index; 		addr_bottom = (3 - level) * stride + wi->pgshift;		index = (ipa & GENMASK_ULL(addr_top, addr_bottom))			>> (addr_bottom - 3); 		paddr = base_addr | index;		ret = read_guest_s2_desc(vcpu, paddr, &desc, wi);		if (ret < 0) {			out->esr = ESR_ELx_FSC_SEA_TTW(level);			return ret;		} 		new_desc = desc; 		/* Check for valid descriptor at this point */		if (!(desc & KVM_PTE_VALID)) {			out->esr = compute_fsc(level, ESR_ELx_FSC_FAULT);			out->desc = desc;			return 1;		} 		if (FIELD_GET(KVM_PTE_TYPE, desc) == KVM_PTE_TYPE_BLOCK) {			if (level < 3)				break; 			out->esr = compute_fsc(level, ESR_ELx_FSC_FAULT);			out->desc = desc;			return 1;		} 		/* We're at the final level */		if (level == 3)			break; 		if (check_output_size(wi, desc)) {			out->esr = compute_fsc(level, ESR_ELx_FSC_ADDRSZ);			out->desc = desc;			return 1;		} 		base_addr = desc & GENMASK_ULL(47, wi->pgshift); 		level += 1;		addr_top = addr_bottom - 1;	} 	if (level < first_block_level) {		out->esr = compute_fsc(level, ESR_ELx_FSC_FAULT);		out->desc = desc;		return 1;	} 	if (check_output_size(wi, desc)) {		out->esr = compute_fsc(level, ESR_ELx_FSC_ADDRSZ);		out->desc = desc;		return 1;	} 	if (wi->ha)		new_desc |= KVM_PTE_LEAF_ATTR_LO_S2_AF; 	if (new_desc != desc) {		ret = swap_guest_s2_desc(vcpu, paddr, desc, new_desc, wi);		if (ret == -EAGAIN)			return ret;		if (ret) {			out->esr = ESR_ELx_FSC_SEA_TTW(level);			out->desc = desc;			return 1;		} 		desc = new_desc;	} 	if (!(desc & KVM_PTE_LEAF_ATTR_LO_S2_AF)) {		out->esr = compute_fsc(level, ESR_ELx_FSC_ACCESS);		out->desc = desc;		return 1;	} 	addr_bottom += contiguous_bit_shift(desc, wi, level); 	/* Calculate and return the result */	paddr = (desc & GENMASK_ULL(47, addr_bottom)) |		(ipa & GENMASK_ULL(addr_bottom - 1, 0));	out->output = paddr;	out->block_size = 1UL << ((3 - level) * stride + wi->pgshift);	out->readable = desc & KVM_PTE_LEAF_ATTR_LO_S2_S2AP_R;	out->writable = desc & KVM_PTE_LEAF_ATTR_LO_S2_S2AP_W;	out->level = level;	out->desc = desc;	return 0;} #define _has_tgran_2(__r, __sz)						\	({								\		u64 _s1, _s2, _mmfr0 = __r;				\									\		_s2 = SYS_FIELD_GET(ID_AA64MMFR0_EL1,			\				    TGRAN##__sz##_2, _mmfr0);		\									\		_s1 = SYS_FIELD_GET(ID_AA64MMFR0_EL1,			\				    TGRAN##__sz, _mmfr0);		\									\		((_s2 != ID_AA64MMFR0_EL1_TGRAN##__sz##_2_NI &&		\		  _s2 != ID_AA64MMFR0_EL1_TGRAN##__sz##_2_TGRAN##__sz) || \		 (_s2 == ID_AA64MMFR0_EL1_TGRAN##__sz##_2_TGRAN##__sz && \		  _s1 != ID_AA64MMFR0_EL1_TGRAN##__sz##_NI));		\	}) static bool has_tgran_2(u64 mmfr0, unsigned int shift){	switch (shift) {	case 12:		return _has_tgran_2(mmfr0, 4);	case 14:		return _has_tgran_2(mmfr0, 16);	case 16:		return _has_tgran_2(mmfr0, 64);	default:		BUG();	}} static unsigned int fallback_tgran2_shift(u64 mmfr0){	if (has_tgran_2(mmfr0, PAGE_SHIFT))		return PAGE_SHIFT;	else if (has_tgran_2(mmfr0, 12))		return 12;	else if (has_tgran_2(mmfr0, 14))		return 14;	else if (has_tgran_2(mmfr0, 16))		return 16;	else		return PAGE_SHIFT;} static unsigned int vtcr_to_tg0_pgshift(struct kvm *kvm, u64 vtcr){	u64 tg0 = FIELD_GET(VTCR_EL2_TG0_MASK, vtcr);	u64 mmfr0 = kvm_read_vm_id_reg(kvm, SYS_ID_AA64MMFR0_EL1);	unsigned int shift; 	switch (tg0) {	case VTCR_EL2_TG0_4K:		shift = 12;		break;	case VTCR_EL2_TG0_16K:		shift = 14;		break;	case VTCR_EL2_TG0_64K:	/* IMPDEF: treat any other value as 64k, subject to fallback */	default:		shift = 16;	} 	/*	 * If TGx is programmed to an unimplemented value (not advertised in	 * ID_AA64MMFR0_EL1), we should treat it as if an implemented value is	 * written, as per the architecture. Choose an available one while	 * prioritizing PAGE_SIZE.	 */	if (!has_tgran_2(mmfr0, shift))		return fallback_tgran2_shift(mmfr0); 	return shift;} static size_t vtcr_to_tg0_pgsize(struct kvm *kvm, u64 vtcr){	return BIT(vtcr_to_tg0_pgshift(kvm, vtcr));} static void setup_s2_walk(struct kvm_vcpu *vcpu, struct s2_walk_info *wi){	u64 vtcr = vcpu_read_sys_reg(vcpu, VTCR_EL2); 	wi->baddr = vcpu_read_sys_reg(vcpu, VTTBR_EL2);	wi->t0sz = vtcr & VTCR_EL2_T0SZ_MASK;	wi->pgshift = vtcr_to_tg0_pgshift(vcpu->kvm, vtcr);	wi->sl = FIELD_GET(VTCR_EL2_SL0_MASK, vtcr);	/* Global limit for now, should eventually be per-VM */	wi->max_oa_bits = min(get_kvm_ipa_limit(),			      ps_to_output_size(FIELD_GET(VTCR_EL2_PS_MASK, vtcr), false));	wi->ha = vtcr & VTCR_EL2_HA;	wi->be = vcpu_read_sys_reg(vcpu, SCTLR_EL2) & SCTLR_ELx_EE;} int kvm_walk_nested_s2(struct kvm_vcpu *vcpu, phys_addr_t gipa,		       struct kvm_s2_trans *result){	struct s2_walk_info wi;	int ret; 	result->esr = 0; 	if (!vcpu_has_nv(vcpu))		return 0; 

Lines 1–500 of 2104. Load more