Commit 708078f6 authored by Chang S. Bae's avatar Chang S. Bae Committed by Thomas Gleixner
Browse files

x86/entry/64: Handle FSGSBASE enabled paranoid entry/exit 



Without FSGSBASE, user space cannot change GSBASE other than through a
PRCTL. The kernel enforces that the user space GSBASE value is postive as
negative values are used for detecting the kernel space GSBASE value in the
paranoid entry code.

If FSGSBASE is enabled, user space can set arbitrary GSBASE values without
kernel intervention, including negative ones, which breaks the paranoid
entry assumptions.

To avoid this, paranoid entry needs to unconditionally save the current
GSBASE value independent of the interrupted context, retrieve and write the
kernel GSBASE and unconditionally restore the saved value on exit. The
restore happens either in paranoid_exit or in the special exit path of the
NMI low level code.

All other entry code pathes which use unconditional SWAPGS are not affected
as they do not depend on the actual content.

[ tglx: Massaged changelogs and comments ]

Suggested-by: default avatarH. Peter Anvin <hpa@zytor.com>
Suggested-by: default avatarAndy Lutomirski <luto@kernel.org>
Suggested-by: default avatarThomas Gleixner <tglx@linutronix.de>
Signed-off-by: default avatarChang S. Bae <chang.seok.bae@intel.com>
Signed-off-by: default avatarThomas Gleixner <tglx@linutronix.de>
Cc: Andi Kleen <ak@linux.intel.com>
Cc: Ravi Shankar <ravi.v.shankar@intel.com>
Cc: Dave Hansen <dave.hansen@linux.intel.com>
Link: https://lkml.kernel.org/r/1557309753-24073-13-git-send-email-chang.seok.bae@intel.com
parent 79e1932f

arch/x86/entry/calling.h

+6 −0
Original line number Diff line number Diff line
@@ -338,6 +338,12 @@ For 32-bit we have the following conventions - kernel is built with 
#endif

.endm



.macro SAVE_AND_SET_GSBASE scratch_reg:req save_reg:req

	rdgsbase \save_reg

	GET_PERCPU_BASE \scratch_reg

	wrgsbase \scratch_reg

.endm



#endif /* CONFIG_X86_64 */



.macro STACKLEAK_ERASE

arch/x86/entry/entry_64.S

+69 −11
Original line number Diff line number Diff line
@@ -38,6 +38,7 @@ 
#include <asm/export.h>

#include <asm/frame.h>

#include <asm/nospec-branch.h>

#include <asm/fsgsbase.h>

#include <linux/err.h>



#include "calling.h"
@@ -947,7 +948,6 @@ ENTRY(\sym) 
	addq	$\ist_offset, CPU_TSS_IST(\shift_ist)

	.endif



	/* these procedures expect "no swapgs" flag in ebx */

	.if \paranoid

	jmp	paranoid_exit

	.else
@@ -1164,9 +1164,14 @@ idtentry machine_check do_mce has_error_code=0 paranoid=1 
#endif



/*

 * Save all registers in pt_regs, and switch gs if needed.

 * Use slow, but surefire "are we in kernel?" check.

 * Return: ebx=0: need swapgs on exit, ebx=1: otherwise

 * Save all registers in pt_regs. Return GSBASE related information

 * in EBX depending on the availability of the FSGSBASE instructions:

 *

 * FSGSBASE	R/EBX

 *     N        0 -> SWAPGS on exit

 *              1 -> no SWAPGS on exit

 *

 *     Y        GSBASE value at entry, must be restored in paranoid_exit

 */

ENTRY(paranoid_entry)

	UNWIND_HINT_FUNC
@@ -1174,7 +1179,6 @@ ENTRY(paranoid_entry) 
	PUSH_AND_CLEAR_REGS save_ret=1

	ENCODE_FRAME_POINTER 8



1:

	/*

	 * Always stash CR3 in %r14.  This value will be restored,

	 * verbatim, at exit.  Needed if paranoid_entry interrupted
@@ -1192,6 +1196,25 @@ ENTRY(paranoid_entry) 
	 */

	SAVE_AND_SWITCH_TO_KERNEL_CR3 scratch_reg=%rax save_reg=%r14



        /*

	 * Handling GSBASE depends on the availability of FSGSBASE.

	 *

	 * Without FSGSBASE the kernel enforces that negative GSBASE

	 * values indicate kernel GSBASE. With FSGSBASE no assumptions

	 * can be made about the GSBASE value when entering from user

	 * space.

	*/

	ALTERNATIVE "jmp .Lparanoid_entry_checkgs", "", X86_FEATURE_FSGSBASE



	/*

	 * Read the current GSBASE and store it in in %rbx unconditionally,

	 * retrieve and set the current CPUs kernel GSBASE. The stored value

	 * has to be restored in paranoid_exit unconditionally.

	 */

	SAVE_AND_SET_GSBASE scratch_reg=%rax save_reg=%rbx

	ret



.Lparanoid_entry_checkgs:

	/* EBX = 1 -> kernel GSBASE active, no restore required */

	movl	$1, %ebx

	/*
@@ -1218,16 +1241,32 @@ END(paranoid_entry) 
 *

 * We may be returning to very strange contexts (e.g. very early

 * in syscall entry), so checking for preemption here would

 * be complicated.  Fortunately, we there's no good reason

 * to try to handle preemption here.

 * be complicated.  Fortunately, there's no good reason to try

 * to handle preemption here.

 *

 * R/EBX contains the GSBASE related information depending on the

 * availability of the FSGSBASE instructions:

 *

 * FSGSBASE	R/EBX

 *     N        0 -> SWAPGS on exit

 *              1 -> no SWAPGS on exit

 *

 * On entry, ebx is "no swapgs" flag (1: don't need swapgs, 0: need it)

 *     Y        User space GSBASE, must be restored unconditionally

 */

ENTRY(paranoid_exit)

	UNWIND_HINT_REGS

	DISABLE_INTERRUPTS(CLBR_ANY)

	TRACE_IRQS_OFF_DEBUG

	/* If EBX is 0, SWAPGS is required */



	/* Handle GS depending on FSGSBASE availability */

	ALTERNATIVE "jmp .Lparanoid_exit_checkgs", "nop",X86_FEATURE_FSGSBASE



	/* With FSGSBASE enabled, unconditionally restore GSBASE */

	wrgsbase	%rbx

	jmp	.Lparanoid_exit_no_swapgs;



.Lparanoid_exit_checkgs:

	/* On non-FSGSBASE systems, conditionally do SWAPGS */

	testl	%ebx, %ebx

	jnz	.Lparanoid_exit_no_swapgs

	TRACE_IRQS_IRETQ
@@ -1235,10 +1274,12 @@ ENTRY(paranoid_exit) 
	RESTORE_CR3	scratch_reg=%rbx save_reg=%r14

	SWAPGS_UNSAFE_STACK

	jmp	.Lparanoid_exit_restore



.Lparanoid_exit_no_swapgs:

	TRACE_IRQS_IRETQ_DEBUG

	/* Always restore stashed CR3 value (see paranoid_entry) */

	RESTORE_CR3	scratch_reg=%rbx save_reg=%r14



.Lparanoid_exit_restore:

	jmp	restore_regs_and_return_to_kernel

END(paranoid_exit)
@@ -1651,10 +1692,27 @@ end_repeat_nmi: 
	/* Always restore stashed CR3 value (see paranoid_entry) */

	RESTORE_CR3 scratch_reg=%r15 save_reg=%r14



	testl	%ebx, %ebx			/* swapgs needed? */

	/*

	 * The above invocation of paranoid_entry stored the GSBASE

	 * related information in R/EBX depending on the availability

	 * of FSGSBASE.

	 *

	 * If FSGSBASE is enabled, restore the saved GSBASE value

	 * unconditionally, otherwise take the conditional SWAPGS path.

	 */

	ALTERNATIVE "jmp nmi_no_fsgsbase", "", X86_FEATURE_FSGSBASE



	wrgsbase	%rbx

	jmp	nmi_restore



nmi_no_fsgsbase:

	/* EBX == 0 -> invoke SWAPGS */

	testl	%ebx, %ebx

	jnz	nmi_restore



nmi_swapgs:

	SWAPGS_UNSAFE_STACK



nmi_restore:

	POP_REGS

CRA Git | Maintained and supported by SUSTech CRA and CCSE