Commit de3cd117 authored by Sean Christopherson's avatar Sean Christopherson Committed by Paolo Bonzini
Browse files

KVM: x86: Omit caching logic for always-available GPRs 



Except for RSP and RIP, which are held in VMX's VMCS, GPRs are always
treated "available and dirtly" on both VMX and SVM, i.e. are
unconditionally loaded/saved immediately before/after VM-Enter/VM-Exit.

Eliminating the unnecessary caching code reduces the size of KVM by a
non-trivial amount, much of which comes from the most common code paths.
E.g. on x86_64, kvm_emulate_cpuid() is reduced from 342 to 182 bytes and
kvm_emulate_hypercall() from 1362 to 1143, with the total size of KVM
dropping by ~1000 bytes.  With CONFIG_RETPOLINE=y, the numbers are even
more pronounced, e.g.: 353->182, 1418->1172 and well over 2000 bytes.

Signed-off-by: default avatarSean Christopherson <sean.j.christopherson@intel.com>
Signed-off-by: default avatarPaolo Bonzini <pbonzini@redhat.com>
parent 0c55671f

arch/x86/kvm/cpuid.c

+6 −6
Original line number Diff line number Diff line
@@ -962,13 +962,13 @@ int kvm_emulate_cpuid(struct kvm_vcpu *vcpu) 
	if (cpuid_fault_enabled(vcpu) && !kvm_require_cpl(vcpu, 0))

		return 1;



	eax = kvm_register_read(vcpu, VCPU_REGS_RAX);

	ecx = kvm_register_read(vcpu, VCPU_REGS_RCX);

	eax = kvm_rax_read(vcpu);

	ecx = kvm_rcx_read(vcpu);

	kvm_cpuid(vcpu, &eax, &ebx, &ecx, &edx, true);

	kvm_register_write(vcpu, VCPU_REGS_RAX, eax);

	kvm_register_write(vcpu, VCPU_REGS_RBX, ebx);

	kvm_register_write(vcpu, VCPU_REGS_RCX, ecx);

	kvm_register_write(vcpu, VCPU_REGS_RDX, edx);

	kvm_rax_write(vcpu, eax);

	kvm_rbx_write(vcpu, ebx);

	kvm_rcx_write(vcpu, ecx);

	kvm_rdx_write(vcpu, edx);

	return kvm_skip_emulated_instruction(vcpu);

}

EXPORT_SYMBOL_GPL(kvm_emulate_cpuid);

arch/x86/kvm/hyperv.c

+12 −12
Original line number Diff line number Diff line
@@ -1526,10 +1526,10 @@ static void kvm_hv_hypercall_set_result(struct kvm_vcpu *vcpu, u64 result) 


	longmode = is_64_bit_mode(vcpu);

	if (longmode)

		kvm_register_write(vcpu, VCPU_REGS_RAX, result);

		kvm_rax_write(vcpu, result);

	else {

		kvm_register_write(vcpu, VCPU_REGS_RDX, result >> 32);

		kvm_register_write(vcpu, VCPU_REGS_RAX, result & 0xffffffff);

		kvm_rdx_write(vcpu, result >> 32);

		kvm_rax_write(vcpu, result & 0xffffffff);

	}

}
@@ -1602,18 +1602,18 @@ int kvm_hv_hypercall(struct kvm_vcpu *vcpu) 
	longmode = is_64_bit_mode(vcpu);



	if (!longmode) {

		param = ((u64)kvm_register_read(vcpu, VCPU_REGS_RDX) << 32) |

			(kvm_register_read(vcpu, VCPU_REGS_RAX) & 0xffffffff);

		ingpa = ((u64)kvm_register_read(vcpu, VCPU_REGS_RBX) << 32) |

			(kvm_register_read(vcpu, VCPU_REGS_RCX) & 0xffffffff);

		outgpa = ((u64)kvm_register_read(vcpu, VCPU_REGS_RDI) << 32) |

			(kvm_register_read(vcpu, VCPU_REGS_RSI) & 0xffffffff);

		param = ((u64)kvm_rdx_read(vcpu) << 32) |

			(kvm_rax_read(vcpu) & 0xffffffff);

		ingpa = ((u64)kvm_rbx_read(vcpu) << 32) |

			(kvm_rcx_read(vcpu) & 0xffffffff);

		outgpa = ((u64)kvm_rdi_read(vcpu) << 32) |

			(kvm_rsi_read(vcpu) & 0xffffffff);

	}

#ifdef CONFIG_X86_64

	else {

		param = kvm_register_read(vcpu, VCPU_REGS_RCX);

		ingpa = kvm_register_read(vcpu, VCPU_REGS_RDX);

		outgpa = kvm_register_read(vcpu, VCPU_REGS_R8);

		param = kvm_rcx_read(vcpu);

		ingpa = kvm_rdx_read(vcpu);

		outgpa = kvm_r8_read(vcpu);

	}

#endif

arch/x86/kvm/kvm_cache_regs.h

+30 −2
Original line number Diff line number Diff line
@@ -9,6 +9,34 @@ 
	(X86_CR4_PVI | X86_CR4_DE | X86_CR4_PCE | X86_CR4_OSFXSR  \

	 | X86_CR4_OSXMMEXCPT | X86_CR4_LA57 | X86_CR4_PGE)



#define BUILD_KVM_GPR_ACCESSORS(lname, uname)				      \

static __always_inline unsigned long kvm_##lname##_read(struct kvm_vcpu *vcpu)\

{									      \

	return vcpu->arch.regs[VCPU_REGS_##uname];			      \

}									      \

static __always_inline void kvm_##lname##_write(struct kvm_vcpu *vcpu,	      \

						unsigned long val)	      \

{									      \

	vcpu->arch.regs[VCPU_REGS_##uname] = val;			      \

}

BUILD_KVM_GPR_ACCESSORS(rax, RAX)

BUILD_KVM_GPR_ACCESSORS(rbx, RBX)

BUILD_KVM_GPR_ACCESSORS(rcx, RCX)

BUILD_KVM_GPR_ACCESSORS(rdx, RDX)

BUILD_KVM_GPR_ACCESSORS(rbp, RBP)

BUILD_KVM_GPR_ACCESSORS(rsi, RSI)

BUILD_KVM_GPR_ACCESSORS(rdi, RDI)

#ifdef CONFIG_X86_64

BUILD_KVM_GPR_ACCESSORS(r8,  R8)

BUILD_KVM_GPR_ACCESSORS(r9,  R9)

BUILD_KVM_GPR_ACCESSORS(r10, R10)

BUILD_KVM_GPR_ACCESSORS(r11, R11)

BUILD_KVM_GPR_ACCESSORS(r12, R12)

BUILD_KVM_GPR_ACCESSORS(r13, R13)

BUILD_KVM_GPR_ACCESSORS(r14, R14)

BUILD_KVM_GPR_ACCESSORS(r15, R15)

#endif



static inline unsigned long kvm_register_read(struct kvm_vcpu *vcpu,

					      enum kvm_reg reg)

{
@@ -83,8 +111,8 @@ static inline ulong kvm_read_cr4(struct kvm_vcpu *vcpu) 


static inline u64 kvm_read_edx_eax(struct kvm_vcpu *vcpu)

{

	return (kvm_register_read(vcpu, VCPU_REGS_RAX) & -1u)

		| ((u64)(kvm_register_read(vcpu, VCPU_REGS_RDX) & -1u) << 32);

	return (kvm_rax_read(vcpu) & -1u)

		| ((u64)(kvm_rdx_read(vcpu) & -1u) << 32);

}



static inline void enter_guest_mode(struct kvm_vcpu *vcpu)

arch/x86/kvm/svm.c

+12 −14
Original line number Diff line number Diff line
@@ -2091,7 +2091,7 @@ static void svm_vcpu_reset(struct kvm_vcpu *vcpu, bool init_event) 
	init_vmcb(svm);



	kvm_cpuid(vcpu, &eax, &dummy, &dummy, &dummy, true);

	kvm_register_write(vcpu, VCPU_REGS_RDX, eax);

	kvm_rdx_write(vcpu, eax);



	if (kvm_vcpu_apicv_active(vcpu) && !init_event)

		avic_update_vapic_bar(svm, APIC_DEFAULT_PHYS_BASE);
@@ -3388,7 +3388,7 @@ static int nested_svm_vmexit(struct vcpu_svm *svm) 
	} else {

		(void)kvm_set_cr3(&svm->vcpu, hsave->save.cr3);

	}

	kvm_register_write(&svm->vcpu, VCPU_REGS_RAX, hsave->save.rax);

	kvm_rax_write(&svm->vcpu, hsave->save.rax);

	kvm_register_write(&svm->vcpu, VCPU_REGS_RSP, hsave->save.rsp);

	kvm_register_write(&svm->vcpu, VCPU_REGS_RIP, hsave->save.rip);

	svm->vmcb->save.dr7 = 0;
@@ -3496,7 +3496,7 @@ static void enter_svm_guest_mode(struct vcpu_svm *svm, u64 vmcb_gpa, 
	kvm_mmu_reset_context(&svm->vcpu);



	svm->vmcb->save.cr2 = svm->vcpu.arch.cr2 = nested_vmcb->save.cr2;

	kvm_register_write(&svm->vcpu, VCPU_REGS_RAX, nested_vmcb->save.rax);

	kvm_rax_write(&svm->vcpu, nested_vmcb->save.rax);

	kvm_register_write(&svm->vcpu, VCPU_REGS_RSP, nested_vmcb->save.rsp);

	kvm_register_write(&svm->vcpu, VCPU_REGS_RIP, nested_vmcb->save.rip);
@@ -3787,11 +3787,11 @@ static int invlpga_interception(struct vcpu_svm *svm) 
{

	struct kvm_vcpu *vcpu = &svm->vcpu;



	trace_kvm_invlpga(svm->vmcb->save.rip, kvm_register_read(&svm->vcpu, VCPU_REGS_RCX),

			  kvm_register_read(&svm->vcpu, VCPU_REGS_RAX));

	trace_kvm_invlpga(svm->vmcb->save.rip, kvm_rcx_read(&svm->vcpu),

			  kvm_rax_read(&svm->vcpu));



	/* Let's treat INVLPGA the same as INVLPG (can be optimized!) */

	kvm_mmu_invlpg(vcpu, kvm_register_read(&svm->vcpu, VCPU_REGS_RAX));

	kvm_mmu_invlpg(vcpu, kvm_rax_read(&svm->vcpu));



	svm->next_rip = kvm_rip_read(&svm->vcpu) + 3;

	return kvm_skip_emulated_instruction(&svm->vcpu);
@@ -3799,7 +3799,7 @@ static int invlpga_interception(struct vcpu_svm *svm) 


static int skinit_interception(struct vcpu_svm *svm)

{

	trace_kvm_skinit(svm->vmcb->save.rip, kvm_register_read(&svm->vcpu, VCPU_REGS_RAX));

	trace_kvm_skinit(svm->vmcb->save.rip, kvm_rax_read(&svm->vcpu));



	kvm_queue_exception(&svm->vcpu, UD_VECTOR);

	return 1;
@@ -3813,7 +3813,7 @@ static int wbinvd_interception(struct vcpu_svm *svm) 
static int xsetbv_interception(struct vcpu_svm *svm)

{

	u64 new_bv = kvm_read_edx_eax(&svm->vcpu);

	u32 index = kvm_register_read(&svm->vcpu, VCPU_REGS_RCX);

	u32 index = kvm_rcx_read(&svm->vcpu);



	if (kvm_set_xcr(&svm->vcpu, index, new_bv) == 0) {

		svm->next_rip = kvm_rip_read(&svm->vcpu) + 3;
@@ -4209,7 +4209,7 @@ static int svm_get_msr(struct kvm_vcpu *vcpu, struct msr_data *msr_info) 


static int rdmsr_interception(struct vcpu_svm *svm)

{

	u32 ecx = kvm_register_read(&svm->vcpu, VCPU_REGS_RCX);

	u32 ecx = kvm_rcx_read(&svm->vcpu);

	struct msr_data msr_info;



	msr_info.index = ecx;
@@ -4221,10 +4221,8 @@ static int rdmsr_interception(struct vcpu_svm *svm) 
	} else {

		trace_kvm_msr_read(ecx, msr_info.data);



		kvm_register_write(&svm->vcpu, VCPU_REGS_RAX,

				   msr_info.data & 0xffffffff);

		kvm_register_write(&svm->vcpu, VCPU_REGS_RDX,

				   msr_info.data >> 32);

		kvm_rax_write(&svm->vcpu, msr_info.data & 0xffffffff);

		kvm_rdx_write(&svm->vcpu, msr_info.data >> 32);

		svm->next_rip = kvm_rip_read(&svm->vcpu) + 2;

		return kvm_skip_emulated_instruction(&svm->vcpu);

	}
@@ -4418,7 +4416,7 @@ static int svm_set_msr(struct kvm_vcpu *vcpu, struct msr_data *msr) 
static int wrmsr_interception(struct vcpu_svm *svm)

{

	struct msr_data msr;

	u32 ecx = kvm_register_read(&svm->vcpu, VCPU_REGS_RCX);

	u32 ecx = kvm_rcx_read(&svm->vcpu);

	u64 data = kvm_read_edx_eax(&svm->vcpu);



	msr.data = data;

arch/x86/kvm/vmx/vmx.c

+1 −1
Original line number Diff line number Diff line
@@ -4926,7 +4926,7 @@ static int handle_wbinvd(struct kvm_vcpu *vcpu) 
static int handle_xsetbv(struct kvm_vcpu *vcpu)

{

	u64 new_bv = kvm_read_edx_eax(vcpu);

	u32 index = kvm_register_read(vcpu, VCPU_REGS_RCX);

	u32 index = kvm_rcx_read(vcpu);



	if (kvm_set_xcr(vcpu, index, new_bv) == 0)

		return kvm_skip_emulated_instruction(vcpu);

CRA Git | Maintained and supported by SUSTech CRA and CCSE