Commit 14c07ad8 authored by Vitaly Kuznetsov's avatar Vitaly Kuznetsov Committed by Paolo Bonzini
Browse files

x86/kvm/mmu: introduce guest_mmu 



When EPT is used for nested guest we need to re-init MMU as shadow
EPT MMU (nested_ept_init_mmu_context() does that). When we return back
from L2 to L1 kvm_mmu_reset_context() in nested_vmx_load_cr3() resets
MMU back to normal TDP mode. Add a special 'guest_mmu' so we can use
separate root caches; the improved hit rate is not very important for
single vCPU performance, but it avoids contention on the mmu_lock for
many vCPUs.

On the nested CPUID benchmark, with 16 vCPUs, an L2->L1->L2 vmexit
goes from 42k to 26k cycles.

Signed-off-by: default avatarVitaly Kuznetsov <vkuznets@redhat.com>
Reviewed-by: default avatarSean Christopherson <sean.j.christopherson@intel.com>
Signed-off-by: default avatarPaolo Bonzini <pbonzini@redhat.com>
parent 6a82cd1c

arch/x86/include/asm/kvm_host.h

+3 −0
Original line number Diff line number Diff line
@@ -548,6 +548,9 @@ struct kvm_vcpu_arch { 
	/* Non-nested MMU for L1 */

	struct kvm_mmu root_mmu;



	/* L1 MMU when running nested */

	struct kvm_mmu guest_mmu;



	/*

	 * Paging state of an L2 guest (used for nested npt)

	 *

arch/x86/kvm/mmu.c

+15 −5
Original line number Diff line number Diff line
@@ -4832,7 +4832,10 @@ EXPORT_SYMBOL_GPL(kvm_init_shadow_mmu); 
static union kvm_mmu_page_role

kvm_calc_shadow_ept_root_page_role(struct kvm_vcpu *vcpu, bool accessed_dirty)

{

	union kvm_mmu_page_role role = vcpu->arch.mmu->base_role;

	union kvm_mmu_page_role role;



	/* Role is inherited from root_mmu */

	role.word = vcpu->arch.root_mmu.base_role.word;



	role.level = PT64_ROOT_4LEVEL;

	role.direct = false;
@@ -4982,8 +4985,10 @@ EXPORT_SYMBOL_GPL(kvm_mmu_load); 


void kvm_mmu_unload(struct kvm_vcpu *vcpu)

{

	kvm_mmu_free_roots(vcpu, vcpu->arch.mmu, KVM_MMU_ROOTS_ALL);

	WARN_ON(VALID_PAGE(vcpu->arch.mmu->root_hpa));

	kvm_mmu_free_roots(vcpu, &vcpu->arch.root_mmu, KVM_MMU_ROOTS_ALL);

	WARN_ON(VALID_PAGE(vcpu->arch.root_mmu.root_hpa));

	kvm_mmu_free_roots(vcpu, &vcpu->arch.guest_mmu, KVM_MMU_ROOTS_ALL);

	WARN_ON(VALID_PAGE(vcpu->arch.guest_mmu.root_hpa));

}

EXPORT_SYMBOL_GPL(kvm_mmu_unload);
@@ -5422,13 +5427,18 @@ int kvm_mmu_create(struct kvm_vcpu *vcpu) 


	vcpu->arch.mmu = &vcpu->arch.root_mmu;

	vcpu->arch.walk_mmu = &vcpu->arch.root_mmu;



	vcpu->arch.root_mmu.root_hpa = INVALID_PAGE;

	vcpu->arch.root_mmu.translate_gpa = translate_gpa;

	vcpu->arch.nested_mmu.translate_gpa = translate_nested_gpa;



	for (i = 0; i < KVM_MMU_NUM_PREV_ROOTS; i++)

		vcpu->arch.root_mmu.prev_roots[i] = KVM_MMU_ROOT_INFO_INVALID;



	vcpu->arch.guest_mmu.root_hpa = INVALID_PAGE;

	vcpu->arch.guest_mmu.translate_gpa = translate_gpa;

	for (i = 0; i < KVM_MMU_NUM_PREV_ROOTS; i++)

		vcpu->arch.guest_mmu.prev_roots[i] = KVM_MMU_ROOT_INFO_INVALID;



	vcpu->arch.nested_mmu.translate_gpa = translate_nested_gpa;

	return alloc_mmu_pages(vcpu);

}

arch/x86/kvm/vmx.c

+22 −13
Original line number Diff line number Diff line
@@ -8423,8 +8423,10 @@ static void vmx_disable_shadow_vmcs(struct vcpu_vmx *vmx) 
	vmcs_write64(VMCS_LINK_POINTER, -1ull);

}



static inline void nested_release_vmcs12(struct vcpu_vmx *vmx)

static inline void nested_release_vmcs12(struct kvm_vcpu *vcpu)

{

	struct vcpu_vmx *vmx = to_vmx(vcpu);



	if (vmx->nested.current_vmptr == -1ull)

		return;
@@ -8438,10 +8440,12 @@ static inline void nested_release_vmcs12(struct vcpu_vmx *vmx) 
	vmx->nested.posted_intr_nv = -1;



	/* Flush VMCS12 to guest memory */

	kvm_vcpu_write_guest_page(&vmx->vcpu,

	kvm_vcpu_write_guest_page(vcpu,

				  vmx->nested.current_vmptr >> PAGE_SHIFT,

				  vmx->nested.cached_vmcs12, 0, VMCS12_SIZE);



	kvm_mmu_free_roots(vcpu, &vcpu->arch.guest_mmu, KVM_MMU_ROOTS_ALL);



	vmx->nested.current_vmptr = -1ull;

}
@@ -8449,8 +8453,10 @@ static inline void nested_release_vmcs12(struct vcpu_vmx *vmx) 
 * Free whatever needs to be freed from vmx->nested when L1 goes down, or

 * just stops using VMX.

 */

static void free_nested(struct vcpu_vmx *vmx)

static void free_nested(struct kvm_vcpu *vcpu)

{

	struct vcpu_vmx *vmx = to_vmx(vcpu);



	if (!vmx->nested.vmxon && !vmx->nested.smm.vmxon)

		return;
@@ -8483,6 +8489,8 @@ static void free_nested(struct vcpu_vmx *vmx) 
		vmx->nested.pi_desc = NULL;

	}



	kvm_mmu_free_roots(vcpu, &vcpu->arch.guest_mmu, KVM_MMU_ROOTS_ALL);



	free_loaded_vmcs(&vmx->nested.vmcs02);

}
@@ -8491,7 +8499,7 @@ static int handle_vmoff(struct kvm_vcpu *vcpu) 
{

	if (!nested_vmx_check_permission(vcpu))

		return 1;

	free_nested(to_vmx(vcpu));

	free_nested(vcpu);

	return nested_vmx_succeed(vcpu);

}
@@ -8517,7 +8525,7 @@ static int handle_vmclear(struct kvm_vcpu *vcpu) 
			VMXERR_VMCLEAR_VMXON_POINTER);



	if (vmptr == vmx->nested.current_vmptr)

		nested_release_vmcs12(vmx);

		nested_release_vmcs12(vcpu);



	kvm_vcpu_write_guest(vcpu,

			vmptr + offsetof(struct vmcs12, launch_state),
@@ -8872,7 +8880,8 @@ static int handle_vmptrld(struct kvm_vcpu *vcpu) 
				VMXERR_VMPTRLD_INCORRECT_VMCS_REVISION_ID);

		}



		nested_release_vmcs12(vmx);

		nested_release_vmcs12(vcpu);



		/*

		 * Load VMCS12 from guest memory since it is not already

		 * cached.
@@ -10928,11 +10937,9 @@ static void vmx_switch_vmcs(struct kvm_vcpu *vcpu, struct loaded_vmcs *vmcs) 
 */

static void vmx_free_vcpu_nested(struct kvm_vcpu *vcpu)

{

       struct vcpu_vmx *vmx = to_vmx(vcpu);



	vcpu_load(vcpu);

       vmx_switch_vmcs(vcpu, &vmx->vmcs01);

       free_nested(vmx);

	vmx_switch_vmcs(vcpu, &to_vmx(vcpu)->vmcs01);

	free_nested(vcpu);

	vcpu_put(vcpu);

}
@@ -11300,6 +11307,7 @@ static void nested_ept_init_mmu_context(struct kvm_vcpu *vcpu) 
{

	WARN_ON(mmu_is_nested(vcpu));



	vcpu->arch.mmu = &vcpu->arch.guest_mmu;

	kvm_init_shadow_ept_mmu(vcpu,

			to_vmx(vcpu)->nested.msrs.ept_caps &

			VMX_EPT_EXECUTE_ONLY_BIT,
@@ -11315,6 +11323,7 @@ static void nested_ept_init_mmu_context(struct kvm_vcpu *vcpu) 


static void nested_ept_uninit_mmu_context(struct kvm_vcpu *vcpu)

{

	vcpu->arch.mmu = &vcpu->arch.root_mmu;

	vcpu->arch.walk_mmu = &vcpu->arch.root_mmu;

}
@@ -13731,7 +13740,7 @@ static void vmx_leave_nested(struct kvm_vcpu *vcpu) 
		to_vmx(vcpu)->nested.nested_run_pending = 0;

		nested_vmx_vmexit(vcpu, -1, 0, 0);

	}

	free_nested(to_vmx(vcpu));

	free_nested(vcpu);

}



static int vmx_check_intercept(struct kvm_vcpu *vcpu,

CRA Git | Maintained and supported by SUSTech CRA and CCSE