Commit 46db09d3 authored by Jan Beulich's avatar Jan Beulich Committed by Ingo Molnar
Browse files

x86-64: Slightly shorten line system call entry and exit paths 



GET_THREAD_INFO() involves a memory read immediately followed by
an "sub" on the value read, in turn (in several cases)
immediately followed by a use of the calculated value as the
base address of a memory access. This combination of
instructions has a non-negligible potential for stalls.

In the system call entry point code, however, the (fixed) offset
of the stack pointer from the end of the stack is generally
known, and hence we can instead avoid the memory load and
subtract, and instead do the memory reference using %rsp as the
base register. To do so in a legible fashion, introduce a
THREAD_INFO() macro which, provided a register (generally %rsp)
and the known offset from the end of the stack, produces a
suitable memory access operand.

The patch attempts to only touch the fast paths (no auditing and
alike), but manages to do so only in the 64-bit entry point
case; the compatibility mode entry points have so many
interdependencies between their various branch targets that it
was necessary to also adjust the slow paths to eliminate the
risk of having missed some register dependency during code
analysis.

Signed-off-by: default avatarJan Beulich <jbeulich@suse.com>
Reviewed-by: default avatarAndi Kleen <ak@linux.intel.com>
Link: http://lkml.kernel.org/r/4ED4CD690200007800064075@nat28.tlf.novell.com


Signed-off-by: default avatarIngo Molnar <mingo@elte.hu>
parent 39e95433

arch/x86/ia32/ia32entry.S

+15 −21
Original line number Diff line number Diff line
@@ -134,7 +134,7 @@ ENTRY(ia32_sysenter_target) 
	CFI_REL_OFFSET rsp,0

	pushfq_cfi

	/*CFI_REL_OFFSET rflags,0*/

	movl	8*3-THREAD_SIZE+TI_sysenter_return(%rsp), %r10d

	movl	TI_sysenter_return+THREAD_INFO(%rsp,3*8-KERNEL_STACK_OFFSET),%r10d

	CFI_REGISTER rip,r10

	pushq_cfi $__USER32_CS

	/*CFI_REL_OFFSET cs,0*/
@@ -150,9 +150,8 @@ ENTRY(ia32_sysenter_target) 
 	.section __ex_table,"a"

 	.quad 1b,ia32_badarg

 	.previous	

	GET_THREAD_INFO(%r10)

	orl    $TS_COMPAT,TI_status(%r10)

	testl  $_TIF_WORK_SYSCALL_ENTRY,TI_flags(%r10)

	orl     $TS_COMPAT,TI_status+THREAD_INFO(%rsp,RIP-ARGOFFSET)

	testl   $_TIF_WORK_SYSCALL_ENTRY,TI_flags+THREAD_INFO(%rsp,RIP-ARGOFFSET)

	CFI_REMEMBER_STATE

	jnz  sysenter_tracesys

	cmpq	$(IA32_NR_syscalls-1),%rax
@@ -162,13 +161,12 @@ sysenter_do_call: 
sysenter_dispatch:

	call	*ia32_sys_call_table(,%rax,8)

	movq	%rax,RAX-ARGOFFSET(%rsp)

	GET_THREAD_INFO(%r10)

	DISABLE_INTERRUPTS(CLBR_NONE)

	TRACE_IRQS_OFF

	testl	$_TIF_ALLWORK_MASK,TI_flags(%r10)

	testl	$_TIF_ALLWORK_MASK,TI_flags+THREAD_INFO(%rsp,RIP-ARGOFFSET)

	jnz	sysexit_audit

sysexit_from_sys_call:

	andl    $~TS_COMPAT,TI_status(%r10)

	andl    $~TS_COMPAT,TI_status+THREAD_INFO(%rsp,RIP-ARGOFFSET)

	/* clear IF, that popfq doesn't enable interrupts early */

	andl  $~0x200,EFLAGS-R11(%rsp) 

	movl	RIP-R11(%rsp),%edx		/* User %eip */
@@ -205,7 +203,7 @@ sysexit_from_sys_call: 
	.endm



	.macro auditsys_exit exit

	testl $(_TIF_ALLWORK_MASK & ~_TIF_SYSCALL_AUDIT),TI_flags(%r10)

	testl $(_TIF_ALLWORK_MASK & ~_TIF_SYSCALL_AUDIT),TI_flags+THREAD_INFO(%rsp,RIP-ARGOFFSET)

	jnz ia32_ret_from_sys_call

	TRACE_IRQS_ON

	sti
@@ -215,12 +213,11 @@ sysexit_from_sys_call: 
	movzbl %al,%edi		/* zero-extend that into %edi */

	inc %edi /* first arg, 0->1(AUDITSC_SUCCESS), 1->2(AUDITSC_FAILURE) */

	call audit_syscall_exit

	GET_THREAD_INFO(%r10)

	movl RAX-ARGOFFSET(%rsp),%eax	/* reload syscall return value */

	movl $(_TIF_ALLWORK_MASK & ~_TIF_SYSCALL_AUDIT),%edi

	cli

	TRACE_IRQS_OFF

	testl %edi,TI_flags(%r10)

	testl %edi,TI_flags+THREAD_INFO(%rsp,RIP-ARGOFFSET)

	jz \exit

	CLEAR_RREGS -ARGOFFSET

	jmp int_with_check
@@ -238,7 +235,7 @@ sysexit_audit: 


sysenter_tracesys:

#ifdef CONFIG_AUDITSYSCALL

	testl	$(_TIF_WORK_SYSCALL_ENTRY & ~_TIF_SYSCALL_AUDIT),TI_flags(%r10)

	testl	$(_TIF_WORK_SYSCALL_ENTRY & ~_TIF_SYSCALL_AUDIT),TI_flags+THREAD_INFO(%rsp,RIP-ARGOFFSET)

	jz	sysenter_auditsys

#endif

	SAVE_REST
@@ -309,9 +306,8 @@ ENTRY(ia32_cstar_target) 
	.section __ex_table,"a"

	.quad 1b,ia32_badarg

	.previous	

	GET_THREAD_INFO(%r10)

	orl   $TS_COMPAT,TI_status(%r10)

	testl $_TIF_WORK_SYSCALL_ENTRY,TI_flags(%r10)

	orl     $TS_COMPAT,TI_status+THREAD_INFO(%rsp,RIP-ARGOFFSET)

	testl   $_TIF_WORK_SYSCALL_ENTRY,TI_flags+THREAD_INFO(%rsp,RIP-ARGOFFSET)

	CFI_REMEMBER_STATE

	jnz   cstar_tracesys

	cmpq $IA32_NR_syscalls-1,%rax
@@ -321,13 +317,12 @@ cstar_do_call: 
cstar_dispatch:

	call *ia32_sys_call_table(,%rax,8)

	movq %rax,RAX-ARGOFFSET(%rsp)

	GET_THREAD_INFO(%r10)

	DISABLE_INTERRUPTS(CLBR_NONE)

	TRACE_IRQS_OFF

	testl $_TIF_ALLWORK_MASK,TI_flags(%r10)

	testl $_TIF_ALLWORK_MASK,TI_flags+THREAD_INFO(%rsp,RIP-ARGOFFSET)

	jnz sysretl_audit

sysretl_from_sys_call:

	andl $~TS_COMPAT,TI_status(%r10)

	andl $~TS_COMPAT,TI_status+THREAD_INFO(%rsp,RIP-ARGOFFSET)

	RESTORE_ARGS 0,-ARG_SKIP,0,0,0

	movl RIP-ARGOFFSET(%rsp),%ecx

	CFI_REGISTER rip,rcx
@@ -355,7 +350,7 @@ sysretl_audit: 


cstar_tracesys:

#ifdef CONFIG_AUDITSYSCALL

	testl $(_TIF_WORK_SYSCALL_ENTRY & ~_TIF_SYSCALL_AUDIT),TI_flags(%r10)

	testl $(_TIF_WORK_SYSCALL_ENTRY & ~_TIF_SYSCALL_AUDIT),TI_flags+THREAD_INFO(%rsp,RIP-ARGOFFSET)

	jz cstar_auditsys

#endif

	xchgl %r9d,%ebp
@@ -420,9 +415,8 @@ ENTRY(ia32_syscall) 
	/* note the registers are not zero extended to the sf.

	   this could be a problem. */

	SAVE_ARGS 0,1,0

	GET_THREAD_INFO(%r10)

	orl   $TS_COMPAT,TI_status(%r10)

	testl $_TIF_WORK_SYSCALL_ENTRY,TI_flags(%r10)

	orl $TS_COMPAT,TI_status+THREAD_INFO(%rsp,RIP-ARGOFFSET)

	testl $_TIF_WORK_SYSCALL_ENTRY,TI_flags+THREAD_INFO(%rsp,RIP-ARGOFFSET)

	jnz ia32_tracesys

	cmpq $(IA32_NR_syscalls-1),%rax

	ja ia32_badsys

arch/x86/include/asm/thread_info.h

+6 −0
Original line number Diff line number Diff line
@@ -232,6 +232,12 @@ static inline struct thread_info *current_thread_info(void) 
	movq PER_CPU_VAR(kernel_stack),reg ; \

	subq $(THREAD_SIZE-KERNEL_STACK_OFFSET),reg



/*

 * Same if PER_CPU_VAR(kernel_stack) is, perhaps with some offset, already in

 * a certain register (to be used in assembler memory operands).

 */

#define THREAD_INFO(reg, off) KERNEL_STACK_OFFSET+(off)-THREAD_SIZE(reg)



#endif



#endif /* !X86_32 */

arch/x86/kernel/entry_64.S

+3 −5
Original line number Diff line number Diff line
@@ -478,8 +478,7 @@ ENTRY(system_call_after_swapgs) 
	movq  %rax,ORIG_RAX-ARGOFFSET(%rsp)

	movq  %rcx,RIP-ARGOFFSET(%rsp)

	CFI_REL_OFFSET rip,RIP-ARGOFFSET

	GET_THREAD_INFO(%rcx)

	testl $_TIF_WORK_SYSCALL_ENTRY,TI_flags(%rcx)

	testl $_TIF_WORK_SYSCALL_ENTRY,TI_flags+THREAD_INFO(%rsp,RIP-ARGOFFSET)

	jnz tracesys

system_call_fastpath:

	cmpq $__NR_syscall_max,%rax
@@ -496,10 +495,9 @@ ret_from_sys_call: 
	/* edi:	flagmask */

sysret_check:

	LOCKDEP_SYS_EXIT

	GET_THREAD_INFO(%rcx)

	DISABLE_INTERRUPTS(CLBR_NONE)

	TRACE_IRQS_OFF

	movl TI_flags(%rcx),%edx

	movl TI_flags+THREAD_INFO(%rsp,RIP-ARGOFFSET),%edx

	andl %edi,%edx

	jnz  sysret_careful

	CFI_REMEMBER_STATE
@@ -583,7 +581,7 @@ sysret_audit: 
	/* Do syscall tracing */

tracesys:

#ifdef CONFIG_AUDITSYSCALL

	testl $(_TIF_WORK_SYSCALL_ENTRY & ~_TIF_SYSCALL_AUDIT),TI_flags(%rcx)

	testl $(_TIF_WORK_SYSCALL_ENTRY & ~_TIF_SYSCALL_AUDIT),TI_flags+THREAD_INFO(%rsp,RIP-ARGOFFSET)

	jz auditsys

#endif

	SAVE_REST

CRA Git | Maintained and supported by SUSTech CRA and CCSE