Commit 117cc7a9 authored by David Woodhouse's avatar David Woodhouse Committed by Thomas Gleixner
Browse files

x86/retpoline: Fill return stack buffer on vmexit 



In accordance with the Intel and AMD documentation, we need to overwrite
all entries in the RSB on exiting a guest, to prevent malicious branch
target predictions from affecting the host kernel. This is needed both
for retpoline and for IBRS.

[ak: numbers again for the RSB stuffing labels]

Signed-off-by: default avatarDavid Woodhouse <dwmw@amazon.co.uk>
Signed-off-by: default avatarThomas Gleixner <tglx@linutronix.de>
Tested-by: default avatarPeter Zijlstra (Intel) <peterz@infradead.org>
Cc: gnomes@lxorguk.ukuu.org.uk
Cc: Rik van Riel <riel@redhat.com>
Cc: Andi Kleen <ak@linux.intel.com>
Cc: Josh Poimboeuf <jpoimboe@redhat.com>
Cc: thomas.lendacky@amd.com
Cc: Linus Torvalds <torvalds@linux-foundation.org>
Cc: Jiri Kosina <jikos@kernel.org>
Cc: Andy Lutomirski <luto@amacapital.net>
Cc: Dave Hansen <dave.hansen@intel.com>
Cc: Kees Cook <keescook@google.com>
Cc: Tim Chen <tim.c.chen@linux.intel.com>
Cc: Greg Kroah-Hartman <gregkh@linux-foundation.org>
Cc: Paul Turner <pjt@google.com>
Link: https://lkml.kernel.org/r/1515755487-8524-1-git-send-email-dwmw@amazon.co.uk
parent 7614e913

arch/x86/include/asm/nospec-branch.h

+77 −1
Original line number Diff line number Diff line
@@ -7,6 +7,48 @@ 
#include <asm/alternative-asm.h>

#include <asm/cpufeatures.h>



/*

 * Fill the CPU return stack buffer.

 *

 * Each entry in the RSB, if used for a speculative 'ret', contains an

 * infinite 'pause; jmp' loop to capture speculative execution.

 *

 * This is required in various cases for retpoline and IBRS-based

 * mitigations for the Spectre variant 2 vulnerability. Sometimes to

 * eliminate potentially bogus entries from the RSB, and sometimes

 * purely to ensure that it doesn't get empty, which on some CPUs would

 * allow predictions from other (unwanted!) sources to be used.

 *

 * We define a CPP macro such that it can be used from both .S files and

 * inline assembly. It's possible to do a .macro and then include that

 * from C via asm(".include <asm/nospec-branch.h>") but let's not go there.

 */



#define RSB_CLEAR_LOOPS		32	/* To forcibly overwrite all entries */

#define RSB_FILL_LOOPS		16	/* To avoid underflow */



/*

 * Google experimented with loop-unrolling and this turned out to be

 * the optimal version — two calls, each with their own speculation

 * trap should their return address end up getting used, in a loop.

 */

#define __FILL_RETURN_BUFFER(reg, nr, sp)	\

	mov	$(nr/2), reg;			\

771:						\

	call	772f;				\

773:	/* speculation trap */			\

	pause;					\

	jmp	773b;				\

772:						\

	call	774f;				\

775:	/* speculation trap */			\

	pause;					\

	jmp	775b;				\

774:						\

	dec	reg;				\

	jnz	771b;				\

	add	$(BITS_PER_LONG/8) * nr, sp;



#ifdef __ASSEMBLY__



/*
@@ -74,6 +116,20 @@ 
#else

	call	*\reg

#endif

.endm



 /*

  * A simpler FILL_RETURN_BUFFER macro. Don't make people use the CPP

  * monstrosity above, manually.

  */

.macro FILL_RETURN_BUFFER reg:req nr:req ftr:req

#ifdef CONFIG_RETPOLINE

	ANNOTATE_NOSPEC_ALTERNATIVE

	ALTERNATIVE "jmp .Lskip_rsb_\@",				\

		__stringify(__FILL_RETURN_BUFFER(\reg,\nr,%_ASM_SP))	\

		\ftr

.Lskip_rsb_\@:

#endif

.endm



#else /* __ASSEMBLY__ */
@@ -119,7 +175,7 @@ 
	X86_FEATURE_RETPOLINE)



# define THUNK_TARGET(addr) [thunk_target] "rm" (addr)

#else /* No retpoline */

#else /* No retpoline for C / inline asm */

# define CALL_NOSPEC "call *%[thunk_target]\n"

# define THUNK_TARGET(addr) [thunk_target] "rm" (addr)

#endif
@@ -134,5 +190,25 @@ enum spectre_v2_mitigation { 
	SPECTRE_V2_IBRS,

};



/*

 * On VMEXIT we must ensure that no RSB predictions learned in the guest

 * can be followed in the host, by overwriting the RSB completely. Both

 * retpoline and IBRS mitigations for Spectre v2 need this; only on future

 * CPUs with IBRS_ATT *might* it be avoided.

 */

static inline void vmexit_fill_RSB(void)

{

#ifdef CONFIG_RETPOLINE

	unsigned long loops = RSB_CLEAR_LOOPS / 2;



	asm volatile (ANNOTATE_NOSPEC_ALTERNATIVE

		      ALTERNATIVE("jmp 910f",

				  __stringify(__FILL_RETURN_BUFFER(%0, RSB_CLEAR_LOOPS, %1)),

				  X86_FEATURE_RETPOLINE)

		      "910:"

		      : "=&r" (loops), ASM_CALL_CONSTRAINT

		      : "r" (loops) : "memory" );

#endif

}

#endif /* __ASSEMBLY__ */

#endif /* __NOSPEC_BRANCH_H__ */

arch/x86/kvm/svm.c

+4 −0
Original line number Diff line number Diff line
@@ -45,6 +45,7 @@ 
#include <asm/debugreg.h>

#include <asm/kvm_para.h>

#include <asm/irq_remapping.h>

#include <asm/nospec-branch.h>



#include <asm/virtext.h>

#include "trace.h"
@@ -4985,6 +4986,9 @@ static void svm_vcpu_run(struct kvm_vcpu *vcpu) 
#endif

		);



	/* Eliminate branch target predictions from guest mode */

	vmexit_fill_RSB();



#ifdef CONFIG_X86_64

	wrmsrl(MSR_GS_BASE, svm->host.gs_base);

#else

arch/x86/kvm/vmx.c

+4 −0
Original line number Diff line number Diff line
@@ -50,6 +50,7 @@ 
#include <asm/apic.h>

#include <asm/irq_remapping.h>

#include <asm/mmu_context.h>

#include <asm/nospec-branch.h>



#include "trace.h"

#include "pmu.h"
@@ -9403,6 +9404,9 @@ static void __noclone vmx_vcpu_run(struct kvm_vcpu *vcpu) 
#endif

	      );



	/* Eliminate branch target predictions from guest mode */

	vmexit_fill_RSB();



	/* MSR_IA32_DEBUGCTLMSR is zeroed on vmexit. Restore it if needed */

	if (debugctlmsr)

		update_debugctlmsr(debugctlmsr);

CRA Git | Maintained and supported by SUSTech CRA and CCSE