Commit 4983e5d7 authored by Thomas Gleixner's avatar Thomas Gleixner
Browse files

x86/entry: Move irq flags tracing to prepare_exit_to_usermode() 



This is another step towards more C-code and less convoluted ASM.

Similar to the entry path, invoke the tracer before context tracking which
might turn off RCU and invoke lockdep as the last step before going back to
user space. Annotate the code sections in exit_to_user_mode() accordingly
so objtool won't complain about the tracer invocation.

Signed-off-by: default avatarThomas Gleixner <tglx@linutronix.de>
Reviewed-by: default avatarAlexandre Chartre <alexandre.chartre@oracle.com>
Acked-by: default avatarPeter Zijlstra <peterz@infradead.org>
Acked-by: default avatarAndy Lutomirski <luto@kernel.org>
Link: https://lkml.kernel.org/r/20200505134340.703783926@linutronix.de
parent dd8e2d9a

arch/x86/entry/common.c

+18 −1
Original line number Diff line number Diff line
@@ -72,10 +72,27 @@ static __always_inline void enter_from_user_mode(void) 
}

#endif



static noinstr void exit_to_user_mode(void)

/**

 * exit_to_user_mode - Fixup state when exiting to user mode

 *

 * Syscall exit enables interrupts, but the kernel state is interrupts

 * disabled when this is invoked. Also tell RCU about it.

 *

 * 1) Trace interrupts on state

 * 2) Invoke context tracking if enabled to adjust RCU state

 * 3) Clear CPU buffers if CPU is affected by MDS and the migitation is on.

 * 4) Tell lockdep that interrupts are enabled

 */

static __always_inline void exit_to_user_mode(void)

{

	instrumentation_begin();

	trace_hardirqs_on_prepare();

	lockdep_hardirqs_on_prepare(CALLER_ADDR0);

	instrumentation_end();



	user_enter_irqoff();

	mds_user_clear_cpu_buffers();

	lockdep_hardirqs_on(CALLER_ADDR0);

}



static void do_audit_syscall_entry(struct pt_regs *regs, u32 arch)

arch/x86/entry/entry_32.S

+4 −8
Original line number Diff line number Diff line
@@ -817,8 +817,7 @@ SYM_CODE_START(ret_from_fork) 
	/* When we fork, we trace the syscall return in the child, too. */

	movl    %esp, %eax

	call    syscall_return_slowpath

	STACKLEAK_ERASE

	jmp     restore_all

	jmp     .Lsyscall_32_done



	/* kernel thread */

1:	movl	%edi, %eax
@@ -862,7 +861,7 @@ ret_from_intr: 
	TRACE_IRQS_OFF

	movl	%esp, %eax

	call	prepare_exit_to_usermode

	jmp	restore_all

	jmp	restore_all_switch_stack

SYM_CODE_END(ret_from_exception)



SYM_ENTRY(__begin_SYSENTER_singlestep_region, SYM_L_GLOBAL, SYM_A_NONE)
@@ -976,7 +975,6 @@ SYM_FUNC_START(entry_SYSENTER_32) 
	STACKLEAK_ERASE



	/* Opportunistic SYSEXIT */

	TRACE_IRQS_ON			/* User mode traces as IRQs on. */



	/*

	 * Setup entry stack - we keep the pointer in %eax and do the
@@ -1079,11 +1077,9 @@ SYM_FUNC_START(entry_INT80_32) 
	movl	%esp, %eax

	call	do_int80_syscall_32

.Lsyscall_32_done:



	STACKLEAK_ERASE



restore_all:

	TRACE_IRQS_ON

restore_all_switch_stack:

	SWITCH_TO_ENTRY_STACK

	CHECK_AND_APPLY_ESPFIX

arch/x86/entry/entry_64.S

+0 −4
Original line number Diff line number Diff line
@@ -172,8 +172,6 @@ SYM_INNER_LABEL(entry_SYSCALL_64_after_hwframe, SYM_L_GLOBAL) 
	movq	%rsp, %rsi

	call	do_syscall_64		/* returns with IRQs disabled */



	TRACE_IRQS_ON			/* return enables interrupts */



	/*

	 * Try to use SYSRET instead of IRET if we're returning to

	 * a completely clean 64-bit userspace context.  If we're not,
@@ -342,7 +340,6 @@ SYM_CODE_START(ret_from_fork) 
	UNWIND_HINT_REGS

	movq	%rsp, %rdi

	call	syscall_return_slowpath	/* returns with IRQs disabled */

	TRACE_IRQS_ON			/* user mode is traced as IRQS on */

	jmp	swapgs_restore_regs_and_return_to_usermode



1:
@@ -620,7 +617,6 @@ ret_from_intr: 
.Lretint_user:

	mov	%rsp,%rdi

	call	prepare_exit_to_usermode

	TRACE_IRQS_ON



SYM_INNER_LABEL(swapgs_restore_regs_and_return_to_usermode, SYM_L_GLOBAL)

#ifdef CONFIG_DEBUG_ENTRY

arch/x86/entry/entry_64_compat.S

+5 −9
Original line number Diff line number Diff line
@@ -132,8 +132,8 @@ SYM_FUNC_START(entry_SYSENTER_compat) 
	movq	%rsp, %rdi

	call	do_fast_syscall_32

	/* XEN PV guests always use IRET path */

	ALTERNATIVE "testl %eax, %eax; jz .Lsyscall_32_done", \

		    "jmp .Lsyscall_32_done", X86_FEATURE_XENPV

	ALTERNATIVE "testl %eax, %eax; jz swapgs_restore_regs_and_return_to_usermode", \

		    "jmp swapgs_restore_regs_and_return_to_usermode", X86_FEATURE_XENPV

	jmp	sysret32_from_system_call



.Lsysenter_fix_flags:
@@ -244,8 +244,8 @@ SYM_INNER_LABEL(entry_SYSCALL_compat_after_hwframe, SYM_L_GLOBAL) 
	movq	%rsp, %rdi

	call	do_fast_syscall_32

	/* XEN PV guests always use IRET path */

	ALTERNATIVE "testl %eax, %eax; jz .Lsyscall_32_done", \

		    "jmp .Lsyscall_32_done", X86_FEATURE_XENPV

	ALTERNATIVE "testl %eax, %eax; jz swapgs_restore_regs_and_return_to_usermode", \

		    "jmp swapgs_restore_regs_and_return_to_usermode", X86_FEATURE_XENPV



	/* Opportunistic SYSRET */

sysret32_from_system_call:
@@ -254,7 +254,7 @@ sysret32_from_system_call: 
	 * stack. So let's erase the thread stack right now.

	 */

	STACKLEAK_ERASE

	TRACE_IRQS_ON			/* User mode traces as IRQs on. */



	movq	RBX(%rsp), %rbx		/* pt_regs->rbx */

	movq	RBP(%rsp), %rbp		/* pt_regs->rbp */

	movq	EFLAGS(%rsp), %r11	/* pt_regs->flags (in r11) */
@@ -393,9 +393,5 @@ SYM_CODE_START(entry_INT80_compat) 


	movq	%rsp, %rdi

	call	do_int80_syscall_32

.Lsyscall_32_done:



	/* Go back to user mode. */

	TRACE_IRQS_ON

	jmp	swapgs_restore_regs_and_return_to_usermode

SYM_CODE_END(entry_INT80_compat)

CRA Git | Maintained and supported by SUSTech CRA and CCSE