Commit 3ebccdf3 authored by Thomas Gleixner's avatar Thomas Gleixner Committed by Paolo Bonzini
Browse files

x86/kvm/vmx: Move guest enter/exit into .noinstr.text 



Move the functions which are inside the RCU off region into the
non-instrumentable text section.

Signed-off-by: default avatarThomas Gleixner <tglx@linutronix.de>
Reviewed-by: default avatarAlexandre Chartre <alexandre.chartre@oracle.com>
Acked-by: default avatarPeter Zijlstra <peterz@infradead.org>
Acked-by: default avatarPaolo Bonzini <pbonzini@redhat.com>

Message-Id: <20200708195322.037311579@linutronix.de>
Signed-off-by: default avatarPaolo Bonzini <pbonzini@redhat.com>
parent 9fc975e9

arch/x86/include/asm/hardirq.h

+2 −2
Original line number Diff line number Diff line
@@ -67,12 +67,12 @@ static inline void kvm_set_cpu_l1tf_flush_l1d(void) 
	__this_cpu_write(irq_stat.kvm_cpu_l1tf_flush_l1d, 1);

}



static inline void kvm_clear_cpu_l1tf_flush_l1d(void)

static __always_inline void kvm_clear_cpu_l1tf_flush_l1d(void)

{

	__this_cpu_write(irq_stat.kvm_cpu_l1tf_flush_l1d, 0);

}



static inline bool kvm_get_cpu_l1tf_flush_l1d(void)

static __always_inline bool kvm_get_cpu_l1tf_flush_l1d(void)

{

	return __this_cpu_read(irq_stat.kvm_cpu_l1tf_flush_l1d);

}

arch/x86/include/asm/kvm_host.h

+8 −0
Original line number Diff line number Diff line
@@ -1598,7 +1598,15 @@ asmlinkage void kvm_spurious_fault(void); 
	insn "\n\t"							\

	"jmp	668f \n\t"						\

	"667: \n\t"							\

	"1: \n\t"							\

	".pushsection .discard.instr_begin \n\t"			\

	".long 1b - . \n\t"						\

	".popsection \n\t"						\

	"call	kvm_spurious_fault \n\t"				\

	"1: \n\t"							\

	".pushsection .discard.instr_end \n\t"				\

	".long 1b - . \n\t"						\

	".popsection \n\t"						\

	"668: \n\t"							\

	_ASM_EXTABLE(666b, 667b)

arch/x86/kvm/vmx/ops.h

+4 −0
Original line number Diff line number Diff line
@@ -146,7 +146,9 @@ do { \ 
			  : : op1 : "cc" : error, fault);		\

	return;								\

error:									\

	instrumentation_begin();					\

	insn##_error(error_args);					\

	instrumentation_end();						\

	return;								\

fault:									\

	kvm_spurious_fault();						\
@@ -161,7 +163,9 @@ do { \ 
			  : : op1, op2 : "cc" : error, fault);		\

	return;								\

error:									\

	instrumentation_begin();					\

	insn##_error(error_args);					\

	instrumentation_end();						\

	return;								\

fault:									\

	kvm_spurious_fault();						\

arch/x86/kvm/vmx/vmenter.S

+4 −1
Original line number Diff line number Diff line
@@ -27,7 +27,7 @@ 
#define VCPU_R15	__VCPU_REGS_R15 * WORD_SIZE

#endif



	.text

.section .noinstr.text, "ax"



/**

 * vmx_vmenter - VM-Enter the current loaded VMCS
@@ -234,6 +234,9 @@ SYM_FUNC_START(__vmx_vcpu_run) 
	jmp 1b

SYM_FUNC_END(__vmx_vcpu_run)





.section .text, "ax"



/**

 * vmread_error_trampoline - Trampoline from inline asm to vmread_error()

 * @field:	VMCS field encoding that failed

arch/x86/kvm/vmx/vmx.c

+62 −49
Original line number Diff line number Diff line
@@ -6116,7 +6116,7 @@ unexpected_vmexit: 
 * information but as all relevant affected CPUs have 32KiB L1D cache size

 * there is no point in doing so.

 */

static void vmx_l1d_flush(struct kvm_vcpu *vcpu)

static noinstr void vmx_l1d_flush(struct kvm_vcpu *vcpu)

{

	int size = PAGE_SIZE << L1D_CACHE_ORDER;
@@ -6149,7 +6149,7 @@ static void vmx_l1d_flush(struct kvm_vcpu *vcpu) 
	vcpu->stat.l1d_flush++;



	if (static_cpu_has(X86_FEATURE_FLUSH_L1D)) {

		wrmsrl(MSR_IA32_FLUSH_CMD, L1D_FLUSH);

		native_wrmsrl(MSR_IA32_FLUSH_CMD, L1D_FLUSH);

		return;

	}
@@ -6635,7 +6635,7 @@ static void vmx_update_hv_timer(struct kvm_vcpu *vcpu) 
	}

}



void vmx_update_host_rsp(struct vcpu_vmx *vmx, unsigned long host_rsp)

void noinstr vmx_update_host_rsp(struct vcpu_vmx *vmx, unsigned long host_rsp)

{

	if (unlikely(host_rsp != vmx->loaded_vmcs->host_state.rsp)) {

		vmx->loaded_vmcs->host_state.rsp = host_rsp;
@@ -6657,6 +6657,63 @@ static fastpath_t vmx_exit_handlers_fastpath(struct kvm_vcpu *vcpu) 


bool __vmx_vcpu_run(struct vcpu_vmx *vmx, unsigned long *regs, bool launched);



static noinstr void vmx_vcpu_enter_exit(struct kvm_vcpu *vcpu,

					struct vcpu_vmx *vmx)

{

	/*

	 * VMENTER enables interrupts (host state), but the kernel state is

	 * interrupts disabled when this is invoked. Also tell RCU about

	 * it. This is the same logic as for exit_to_user_mode().

	 *

	 * This ensures that e.g. latency analysis on the host observes

	 * guest mode as interrupt enabled.

	 *

	 * guest_enter_irqoff() informs context tracking about the

	 * transition to guest mode and if enabled adjusts RCU state

	 * accordingly.

	 */

	instrumentation_begin();

	trace_hardirqs_on_prepare();

	lockdep_hardirqs_on_prepare(CALLER_ADDR0);

	instrumentation_end();



	guest_enter_irqoff();

	lockdep_hardirqs_on(CALLER_ADDR0);



	/* L1D Flush includes CPU buffer clear to mitigate MDS */

	if (static_branch_unlikely(&vmx_l1d_should_flush))

		vmx_l1d_flush(vcpu);

	else if (static_branch_unlikely(&mds_user_clear))

		mds_clear_cpu_buffers();



	if (vcpu->arch.cr2 != read_cr2())

		write_cr2(vcpu->arch.cr2);



	vmx->fail = __vmx_vcpu_run(vmx, (unsigned long *)&vcpu->arch.regs,

				   vmx->loaded_vmcs->launched);



	vcpu->arch.cr2 = read_cr2();



	/*

	 * VMEXIT disables interrupts (host state), but tracing and lockdep

	 * have them in state 'on' as recorded before entering guest mode.

	 * Same as enter_from_user_mode().

	 *

	 * guest_exit_irqoff() restores host context and reinstates RCU if

	 * enabled and required.

	 *

	 * This needs to be done before the below as native_read_msr()

	 * contains a tracepoint and x86_spec_ctrl_restore_host() calls

	 * into world and some more.

	 */

	lockdep_hardirqs_off(CALLER_ADDR0);

	guest_exit_irqoff();



	instrumentation_begin();

	trace_hardirqs_off_finish();

	instrumentation_end();

}



static fastpath_t vmx_vcpu_run(struct kvm_vcpu *vcpu)

{

	fastpath_t exit_fastpath;
@@ -6731,52 +6788,8 @@ reenter_guest: 
	 */

	x86_spec_ctrl_set_guest(vmx->spec_ctrl, 0);



	/*

	 * VMENTER enables interrupts (host state), but the kernel state is

	 * interrupts disabled when this is invoked. Also tell RCU about

	 * it. This is the same logic as for exit_to_user_mode().

	 *

	 * This ensures that e.g. latency analysis on the host observes

	 * guest mode as interrupt enabled.

	 *

	 * guest_enter_irqoff() informs context tracking about the

	 * transition to guest mode and if enabled adjusts RCU state

	 * accordingly.

	 */

	trace_hardirqs_on_prepare();

	lockdep_hardirqs_on_prepare(CALLER_ADDR0);

	guest_enter_irqoff();

	lockdep_hardirqs_on(CALLER_ADDR0);



	/* L1D Flush includes CPU buffer clear to mitigate MDS */

	if (static_branch_unlikely(&vmx_l1d_should_flush))

		vmx_l1d_flush(vcpu);

	else if (static_branch_unlikely(&mds_user_clear))

		mds_clear_cpu_buffers();



	if (vcpu->arch.cr2 != read_cr2())

		write_cr2(vcpu->arch.cr2);



	vmx->fail = __vmx_vcpu_run(vmx, (unsigned long *)&vcpu->arch.regs,

				   vmx->loaded_vmcs->launched);



	vcpu->arch.cr2 = read_cr2();



	/*

	 * VMEXIT disables interrupts (host state), but tracing and lockdep

	 * have them in state 'on' as recorded before entering guest mode.

	 * Same as enter_from_user_mode().

	 *

	 * guest_exit_irqoff() restores host context and reinstates RCU if

	 * enabled and required.

	 *

	 * This needs to be done before the below as native_read_msr()

	 * contains a tracepoint and x86_spec_ctrl_restore_host() calls

	 * into world and some more.

	 */

	lockdep_hardirqs_off(CALLER_ADDR0);

	guest_exit_irqoff();

	trace_hardirqs_off_finish();

	/* The actual VMENTER/EXIT is in the .noinstr.text section. */

	vmx_vcpu_enter_exit(vcpu, vmx);



	/*

	 * We do not use IBRS in the kernel. If this vCPU has used the

CRA Git | Maintained and supported by SUSTech CRA and CCSE