Commit cc1ac9c7 authored by Peter Zijlstra's avatar Peter Zijlstra
Browse files

x86/retpoline: Fix retpoline unwind 



Currently objtool cannot understand retpolines, and thus cannot
generate ORC unwind information for them. This means that we cannot
unwind from the middle of a retpoline.

The recent ANNOTATE_INTRA_FUNCTION_CALL and UNWIND_HINT_RET_OFFSET
support in objtool enables it to understand the basic retpoline
construct. A further problem is that the ORC unwind information is
alternative invariant; IOW. every alternative should have the same
ORC, retpolines obviously violate this. This means we need to
out-of-line them.

Since all GCC generated code already uses out-of-line retpolines, this
should not affect performance much, if anything.

This will enable objtool to generate valid ORC data for the
out-of-line copies, which means we can correctly and reliably unwind
through a retpoline.

Signed-off-by: default avatarPeter Zijlstra (Intel) <peterz@infradead.org>
Acked-by: default avatarJosh Poimboeuf <jpoimboe@redhat.com>
Link: https://lkml.kernel.org/r/20200428191700.210835357@infradead.org
parent 34fdce69

arch/x86/include/asm/asm-prototypes.h

+7 −0
Original line number Diff line number Diff line
@@ -21,8 +21,15 @@ extern void cmpxchg8b_emu(void); 
#define DECL_INDIRECT_THUNK(reg) \

	extern asmlinkage void __x86_indirect_thunk_ ## reg (void);



#define DECL_RETPOLINE(reg) \

	extern asmlinkage void __x86_retpoline_ ## reg (void);



#undef GEN

#define GEN(reg) DECL_INDIRECT_THUNK(reg)

#include <asm/GEN-for-each-reg.h>



#undef GEN

#define GEN(reg) DECL_RETPOLINE(reg)

#include <asm/GEN-for-each-reg.h>



#endif /* CONFIG_RETPOLINE */

arch/x86/include/asm/nospec-branch.h

+8 −48
Original line number Diff line number Diff line
@@ -12,15 +12,6 @@ 
#include <asm/msr-index.h>

#include <asm/unwind_hints.h>



/*

 * This should be used immediately before a retpoline alternative. It tells

 * objtool where the retpolines are so that it can make sense of the control

 * flow by just reading the original instruction(s) and ignoring the

 * alternatives.

 */

#define ANNOTATE_NOSPEC_ALTERNATIVE \

	ANNOTATE_IGNORE_ALTERNATIVE



/*

 * Fill the CPU return stack buffer.

 *
@@ -82,34 +73,6 @@ 
	.popsection

.endm



/*

 * These are the bare retpoline primitives for indirect jmp and call.

 * Do not use these directly; they only exist to make the ALTERNATIVE

 * invocation below less ugly.

 */

.macro RETPOLINE_JMP reg:req

	call	.Ldo_rop_\@

.Lspec_trap_\@:

	pause

	lfence

	jmp	.Lspec_trap_\@

.Ldo_rop_\@:

	mov	\reg, (%_ASM_SP)

	ret

.endm



/*

 * This is a wrapper around RETPOLINE_JMP so the called function in reg

 * returns to the instruction after the macro.

 */

.macro RETPOLINE_CALL reg:req

	jmp	.Ldo_call_\@

.Ldo_retpoline_jmp_\@:

	RETPOLINE_JMP \reg

.Ldo_call_\@:

	call	.Ldo_retpoline_jmp_\@

.endm



/*

 * JMP_NOSPEC and CALL_NOSPEC macros can be used instead of a simple

 * indirect jmp/call which may be susceptible to the Spectre variant 2
@@ -117,9 +80,8 @@ 
 */

.macro JMP_NOSPEC reg:req

#ifdef CONFIG_RETPOLINE

	ANNOTATE_NOSPEC_ALTERNATIVE

	ALTERNATIVE_2 __stringify(ANNOTATE_RETPOLINE_SAFE; jmp *%\reg), \

		__stringify(RETPOLINE_JMP %\reg), X86_FEATURE_RETPOLINE,\

		      __stringify(jmp __x86_retpoline_\reg), X86_FEATURE_RETPOLINE, \

		      __stringify(lfence; ANNOTATE_RETPOLINE_SAFE; jmp *%\reg), X86_FEATURE_RETPOLINE_AMD

#else

	jmp	*%\reg
@@ -128,9 +90,8 @@ 


.macro CALL_NOSPEC reg:req

#ifdef CONFIG_RETPOLINE

	ANNOTATE_NOSPEC_ALTERNATIVE

	ALTERNATIVE_2 __stringify(ANNOTATE_RETPOLINE_SAFE; call *%\reg), \

		__stringify(RETPOLINE_CALL %\reg), X86_FEATURE_RETPOLINE,\

		      __stringify(call __x86_retpoline_\reg), X86_FEATURE_RETPOLINE, \

		      __stringify(lfence; ANNOTATE_RETPOLINE_SAFE; call *%\reg), X86_FEATURE_RETPOLINE_AMD

#else

	call	*%\reg
@@ -165,16 +126,16 @@ 
 * which is ensured when CONFIG_RETPOLINE is defined.

 */

# define CALL_NOSPEC						\

	ANNOTATE_NOSPEC_ALTERNATIVE				\

	ALTERNATIVE_2(						\

	ANNOTATE_RETPOLINE_SAFE					\

	"call *%[thunk_target]\n",				\

	"call __x86_indirect_thunk_%V[thunk_target]\n",		\

	"call __x86_retpoline_%V[thunk_target]\n",		\

	X86_FEATURE_RETPOLINE,					\

	"lfence;\n"						\

	ANNOTATE_RETPOLINE_SAFE					\

	"call *%[thunk_target]\n",				\

	X86_FEATURE_RETPOLINE_AMD)



# define THUNK_TARGET(addr) [thunk_target] "r" (addr)



#else /* CONFIG_X86_32 */
@@ -184,7 +145,6 @@ 
 * here, anyway.

 */

# define CALL_NOSPEC						\

	ANNOTATE_NOSPEC_ALTERNATIVE				\

	ALTERNATIVE_2(						\

	ANNOTATE_RETPOLINE_SAFE					\

	"call *%[thunk_target]\n",				\

arch/x86/lib/retpoline.S

+23 −3
Original line number Diff line number Diff line
@@ -7,15 +7,31 @@ 
#include <asm/alternative-asm.h>

#include <asm/export.h>

#include <asm/nospec-branch.h>

#include <asm/unwind_hints.h>

#include <asm/frame.h>



.macro THUNK reg

	.section .text.__x86.indirect_thunk



	.align 32

SYM_FUNC_START(__x86_indirect_thunk_\reg)

	CFI_STARTPROC

	JMP_NOSPEC %\reg

	CFI_ENDPROC

	JMP_NOSPEC \reg

SYM_FUNC_END(__x86_indirect_thunk_\reg)



SYM_FUNC_START_NOALIGN(__x86_retpoline_\reg)

	ANNOTATE_INTRA_FUNCTION_CALL

	call	.Ldo_rop_\@

.Lspec_trap_\@:

	UNWIND_HINT_EMPTY

	pause

	lfence

	jmp	.Lspec_trap_\@

.Ldo_rop_\@:

	mov	%\reg, (%_ASM_SP)

	UNWIND_HINT_RET_OFFSET

	ret

SYM_FUNC_END(__x86_retpoline_\reg)



.endm



/*
@@ -32,6 +48,7 @@ SYM_FUNC_END(__x86_indirect_thunk_\reg) 


#define __EXPORT_THUNK(sym)	_ASM_NOKPROBE(sym); EXPORT_SYMBOL(sym)

#define EXPORT_THUNK(reg)	__EXPORT_THUNK(__x86_indirect_thunk_ ## reg)

#define EXPORT_RETPOLINE(reg)  __EXPORT_THUNK(__x86_retpoline_ ## reg)



#undef GEN

#define GEN(reg) THUNK reg
@@ -41,3 +58,6 @@ SYM_FUNC_END(__x86_indirect_thunk_\reg) 
#define GEN(reg) EXPORT_THUNK(reg)

#include <asm/GEN-for-each-reg.h>



#undef GEN

#define GEN(reg) EXPORT_RETPOLINE(reg)

#include <asm/GEN-for-each-reg.h>

CRA Git | Maintained and supported by SUSTech CRA and CCSE