Commit 96b6352c authored by Andy Lutomirski's avatar Andy Lutomirski
Browse files

x86_64, entry: Remove the syscall exit audit and schedule optimizations 

We used to optimize rescheduling and audit on syscall exit.  Now
that the full slow path is reasonably fast, remove these
optimizations.  Syscall exit auditing is now handled exclusively by
syscall_trace_leave.

This adds something like 10ns to the previously optimized paths on
my computer, presumably due mostly to SAVE_REST / RESTORE_REST.

I think that we should eventually replace both the syscall and
non-paranoid interrupt exit slow paths with a pair of C functions
along the lines of the syscall entry hooks.

Link: http://lkml.kernel.org/r/22f2aa4a0361707a5cfb1de9d45260b39965dead.1421453410.git.luto@amacapital.net


Acked-by: default avatarBorislav Petkov <bp@suse.de>
Signed-off-by: default avatarAndy Lutomirski <luto@amacapital.net>
parent 2a23c6b8

arch/x86/kernel/entry_64.S

+5 −47
Original line number Diff line number Diff line
@@ -361,15 +361,12 @@ system_call_fastpath: 
 * Has incomplete stack frame and undefined top of stack.

 */

ret_from_sys_call:

	movl $_TIF_ALLWORK_MASK,%edi

	/* edi:	flagmask */

sysret_check:

	testl $_TIF_ALLWORK_MASK,TI_flags+THREAD_INFO(%rsp,RIP-ARGOFFSET)

	jnz int_ret_from_sys_call_fixup	/* Go the the slow path */



	LOCKDEP_SYS_EXIT

	DISABLE_INTERRUPTS(CLBR_NONE)

	TRACE_IRQS_OFF

	movl TI_flags+THREAD_INFO(%rsp,RIP-ARGOFFSET),%edx

	andl %edi,%edx

	jnz  sysret_careful

	CFI_REMEMBER_STATE

	/*

	 * sysretq will re-enable interrupts:
@@ -383,49 +380,10 @@ sysret_check: 
	USERGS_SYSRET64



	CFI_RESTORE_STATE

	/* Handle reschedules */

	/* edx:	work, edi: workmask */

sysret_careful:

	bt $TIF_NEED_RESCHED,%edx

	jnc sysret_signal

	TRACE_IRQS_ON

	ENABLE_INTERRUPTS(CLBR_NONE)

	pushq_cfi %rdi

	SCHEDULE_USER

	popq_cfi %rdi

	jmp sysret_check



	/* Handle a signal */

sysret_signal:

	TRACE_IRQS_ON

	ENABLE_INTERRUPTS(CLBR_NONE)

#ifdef CONFIG_AUDITSYSCALL

	bt $TIF_SYSCALL_AUDIT,%edx

	jc sysret_audit

#endif

	/*

	 * We have a signal, or exit tracing or single-step.

	 * These all wind up with the iret return path anyway,

	 * so just join that path right now.

	 */

int_ret_from_sys_call_fixup:

	FIXUP_TOP_OF_STACK %r11, -ARGOFFSET

	jmp int_check_syscall_exit_work



#ifdef CONFIG_AUDITSYSCALL

	/*

	 * Return fast path for syscall audit.  Call __audit_syscall_exit()

	 * directly and then jump back to the fast path with TIF_SYSCALL_AUDIT

	 * masked off.

	 */

sysret_audit:

	movq RAX-ARGOFFSET(%rsp),%rsi	/* second arg, syscall return value */

	cmpq $-MAX_ERRNO,%rsi	/* is it < -MAX_ERRNO? */

	setbe %al		/* 1 if so, 0 if not */

	movzbl %al,%edi		/* zero-extend that into %edi */

	call __audit_syscall_exit

	movl $(_TIF_ALLWORK_MASK & ~_TIF_SYSCALL_AUDIT),%edi

	jmp sysret_check

#endif	/* CONFIG_AUDITSYSCALL */

	jmp int_ret_from_sys_call



	/* Do syscall tracing */

tracesys:

CRA Git | Maintained and supported by SUSTech CRA and CCSE