Commit e2f6bc25 authored by Alexander van Heukelum's avatar Alexander van Heukelum Committed by Ingo Molnar
Browse files

x86: entry_64.S: factor out save_paranoid and paranoid_exit 



Impact: cleanup, shrink kernel image size

Also expand the paranoid_exit0 macro into nmi_exit inside the
nmi stub in the case of enabled irq-tracing.

This gives a few hundred bytes code size reduction.

Signed-off-by: default avatarAlexander van Heukelum <heukelum@fastmail.fm>
Signed-off-by: default avatarIngo Molnar <mingo@elte.hu>
parent c002a1e6

arch/x86/kernel/entry_64.S

+102 −49
Original line number Diff line number Diff line
@@ -353,6 +353,36 @@ ENTRY(save_rest) 
	CFI_ENDPROC

END(save_rest)



/* save complete stack frame */

ENTRY(save_paranoid)

	XCPT_FRAME 1 RDI+8

	cld

	movq_cfi rdi, RDI+8

	movq_cfi rsi, RSI+8

	movq_cfi rdx, RDX+8

	movq_cfi rcx, RCX+8

	movq_cfi rax, RAX+8

	movq_cfi r8, R8+8

	movq_cfi r9, R9+8

	movq_cfi r10, R10+8

	movq_cfi r11, R11+8

	movq_cfi rbx, RBX+8

	movq_cfi rbp, RBP+8

	movq_cfi r12, R12+8

	movq_cfi r13, R13+8

	movq_cfi r14, R14+8

	movq_cfi r15, R15+8

	movl $1,%ebx

	movl $MSR_GS_BASE,%ecx

	rdmsr

	testl %edx,%edx

	js 1f	/* negative -> in kernel */

	SWAPGS

	xorl %ebx,%ebx

1:	ret

	CFI_ENDPROC

END(save_paranoid)



/*

 * A newly forked process directly context switches into this.

 */
@@ -1012,24 +1042,15 @@ END(spurious_interrupt) 
	.endm



	/* error code is on the stack already */

	/* handle NMI like exceptions that can happen everywhere */

	.macro paranoidentry sym, ist=0, irqtrace=1

	SAVE_ALL

	cld

	movl $1,%ebx

	movl  $MSR_GS_BASE,%ecx

	rdmsr

	testl %edx,%edx

	js    1f

	SWAPGS

	xorl  %ebx,%ebx

1:

	.macro paranoidentry sym ist=0

	subq $15*8, %rsp

	CFI_ADJUST_CFA_OFFSET 15*8

	call save_paranoid

	DEFAULT_FRAME 0

	.if \ist

	movq	%gs:pda_data_offset, %rbp

	.endif

	.if \irqtrace

	TRACE_IRQS_OFF

	.endif

	movq %rsp,%rdi

	movq ORIG_RAX(%rsp),%rsi

	movq $-1,ORIG_RAX(%rsp)
@@ -1041,9 +1062,7 @@ END(spurious_interrupt) 
	addq	$EXCEPTION_STKSZ, per_cpu__init_tss + TSS_ist + (\ist - 1) * 8(%rbp)

	.endif

	DISABLE_INTERRUPTS(CLBR_NONE)

	.if \irqtrace

	TRACE_IRQS_OFF

	.endif

	.endm



	/*
@@ -1058,57 +1077,48 @@ END(spurious_interrupt) 
	 * is fundamentally NMI-unsafe. (we cannot change the soft and

	 * hard flags at once, atomically)

	 */

	.macro paranoidexit trace=1



	/* ebx:	no swapgs flag */

paranoid_exit\trace:

KPROBE_ENTRY(paranoid_exit)

	INTR_FRAME

	testl %ebx,%ebx				/* swapgs needed? */

	jnz paranoid_restore\trace

	jnz paranoid_restore

	testl $3,CS(%rsp)

	jnz   paranoid_userspace\trace

paranoid_swapgs\trace:

	.if \trace

	jnz   paranoid_userspace

paranoid_swapgs:

	TRACE_IRQS_IRETQ 0

	.endif

	SWAPGS_UNSAFE_STACK

paranoid_restore\trace:

paranoid_restore:

	RESTORE_ALL 8

	jmp irq_return

paranoid_userspace\trace:

paranoid_userspace:

	GET_THREAD_INFO(%rcx)

	movl TI_flags(%rcx),%ebx

	andl $_TIF_WORK_MASK,%ebx

	jz paranoid_swapgs\trace

	jz paranoid_swapgs

	movq %rsp,%rdi			/* &pt_regs */

	call sync_regs

	movq %rax,%rsp			/* switch stack for scheduling */

	testl $_TIF_NEED_RESCHED,%ebx

	jnz paranoid_schedule\trace

	jnz paranoid_schedule

	movl %ebx,%edx			/* arg3: thread flags */

	.if \trace

	TRACE_IRQS_ON

	.endif

	ENABLE_INTERRUPTS(CLBR_NONE)

	xorl %esi,%esi 			/* arg2: oldset */

	movq %rsp,%rdi 			/* arg1: &pt_regs */

	call do_notify_resume

	DISABLE_INTERRUPTS(CLBR_NONE)

	.if \trace

	TRACE_IRQS_OFF

	.endif

	jmp paranoid_userspace\trace

paranoid_schedule\trace:

	.if \trace

	jmp paranoid_userspace

paranoid_schedule:

	TRACE_IRQS_ON

	.endif

	ENABLE_INTERRUPTS(CLBR_ANY)

	call schedule

	DISABLE_INTERRUPTS(CLBR_ANY)

	.if \trace

	TRACE_IRQS_OFF

	.endif

	jmp paranoid_userspace\trace

	jmp paranoid_userspace

	CFI_ENDPROC

	.endm

END(paranoid_exit)



/*

 * Exception entry point. This expects an error code/orig_rax on the stack.
@@ -1326,20 +1336,63 @@ KPROBE_ENTRY(debug) 
	pushq $0

	CFI_ADJUST_CFA_OFFSET 8

	paranoidentry do_debug, DEBUG_STACK

	paranoidexit

	jmp paranoid_exit

 	CFI_ENDPROC

KPROBE_END(debug)



	/* runs on exception stack */

KPROBE_ENTRY(nmi)

	INTR_FRAME

	PARAVIRT_ADJUST_EXCEPTION_FRAME

	pushq $-1

	CFI_ADJUST_CFA_OFFSET 8

	paranoidentry do_nmi, 0, 0

	pushq_cfi $-1

	subq $15*8, %rsp

	CFI_ADJUST_CFA_OFFSET 15*8

	call save_paranoid

	DEFAULT_FRAME 0

	/* paranoidentry do_nmi, 0; without TRACE_IRQS_OFF */

	movq %rsp,%rdi

	movq ORIG_RAX(%rsp),%rsi

	movq $-1,ORIG_RAX(%rsp)

	call do_nmi

	DISABLE_INTERRUPTS(CLBR_NONE)

#ifdef CONFIG_TRACE_IRQFLAGS

	paranoidexit 0

	/* paranoidexit; without TRACE_IRQS_OFF */

	/* ebx:	no swapgs flag */

nmi_exit:

	testl %ebx,%ebx				/* swapgs needed? */

	jnz nmi_restore

	testl $3,CS(%rsp)

	jnz nmi_userspace

nmi_swapgs:

	SWAPGS_UNSAFE_STACK

nmi_restore:

	RESTORE_ALL 8

	jmp irq_return

nmi_userspace:

	GET_THREAD_INFO(%rcx)

	movl TI_flags(%rcx),%ebx

	andl $_TIF_WORK_MASK,%ebx

	jz nmi_swapgs

	movq %rsp,%rdi			/* &pt_regs */

	call sync_regs

	movq %rax,%rsp			/* switch stack for scheduling */

	testl $_TIF_NEED_RESCHED,%ebx

	jnz nmi_schedule

	movl %ebx,%edx			/* arg3: thread flags */

	ENABLE_INTERRUPTS(CLBR_NONE)

	xorl %esi,%esi 			/* arg2: oldset */

	movq %rsp,%rdi 			/* arg1: &pt_regs */

	call do_notify_resume

	DISABLE_INTERRUPTS(CLBR_NONE)

	jmp nmi_userspace

nmi_schedule:

	ENABLE_INTERRUPTS(CLBR_ANY)

	call schedule

	DISABLE_INTERRUPTS(CLBR_ANY)

	jmp nmi_userspace

	CFI_ENDPROC

#else

	jmp paranoid_exit1

	jmp paranoid_exit

 	CFI_ENDPROC

#endif

KPROBE_END(nmi)
@@ -1350,7 +1403,7 @@ KPROBE_ENTRY(int3) 
 	pushq $0

 	CFI_ADJUST_CFA_OFFSET 8

 	paranoidentry do_int3, DEBUG_STACK

 	jmp paranoid_exit1

 	jmp paranoid_exit

 	CFI_ENDPROC

KPROBE_END(int3)
@@ -1375,7 +1428,7 @@ ENTRY(double_fault) 
	XCPT_FRAME

	PARAVIRT_ADJUST_EXCEPTION_FRAME

	paranoidentry do_double_fault

	jmp paranoid_exit1

	jmp paranoid_exit

	CFI_ENDPROC

END(double_fault)
@@ -1392,7 +1445,7 @@ ENTRY(stack_segment) 
	XCPT_FRAME

	PARAVIRT_ADJUST_EXCEPTION_FRAME

	paranoidentry do_stack_segment

	jmp paranoid_exit1

	jmp paranoid_exit

	CFI_ENDPROC

END(stack_segment)
@@ -1420,7 +1473,7 @@ ENTRY(machine_check) 
	pushq $0

	CFI_ADJUST_CFA_OFFSET 8

	paranoidentry do_machine_check

	jmp paranoid_exit1

	jmp paranoid_exit

	CFI_ENDPROC

END(machine_check)

#endif

CRA Git | Maintained and supported by SUSTech CRA and CCSE