Commit 72fe4858 authored by Glauber de Oliveira Costa's avatar Glauber de Oliveira Costa Committed by Ingo Molnar
Browse files

x86: replace privileged instructions with paravirt macros 



The assembly code in entry_64.S issues a bunch of privileged instructions,
like cli, sti, swapgs, and others. Paravirt guests are forbidden to do so,
and we then replace them with macros that will do the right thing.

Signed-off-by: default avatarGlauber de Oliveira Costa <gcosta@redhat.com>
Signed-off-by: default avatarIngo Molnar <mingo@elte.hu>
Signed-off-by: default avatarThomas Gleixner <tglx@linutronix.de>
parent e801f864

arch/x86/kernel/entry_64.S

+59 −42
Original line number Diff line number Diff line
@@ -50,6 +50,7 @@ 
#include <asm/hw_irq.h>

#include <asm/page.h>

#include <asm/irqflags.h>

#include <asm/paravirt.h>



	.code64
@@ -57,6 +58,13 @@ 
#define retint_kernel retint_restore_args

#endif	



#ifdef CONFIG_PARAVIRT

ENTRY(native_irq_enable_syscall_ret)

	movq	%gs:pda_oldrsp,%rsp

	swapgs

	sysretq

#endif /* CONFIG_PARAVIRT */





.macro TRACE_IRQS_IRETQ offset=ARGOFFSET

#ifdef CONFIG_TRACE_IRQFLAGS
@@ -216,14 +224,21 @@ ENTRY(system_call) 
	CFI_DEF_CFA	rsp,PDA_STACKOFFSET

	CFI_REGISTER	rip,rcx

	/*CFI_REGISTER	rflags,r11*/

	swapgs

	SWAPGS_UNSAFE_STACK

	/*

	 * A hypervisor implementation might want to use a label

	 * after the swapgs, so that it can do the swapgs

	 * for the guest and jump here on syscall.

	 */

ENTRY(system_call_after_swapgs)



	movq	%rsp,%gs:pda_oldrsp 

	movq	%gs:pda_kernelstack,%rsp

	/*

	 * No need to follow this irqs off/on section - it's straight

	 * and short:

	 */

	sti					

	ENABLE_INTERRUPTS(CLBR_NONE)

	SAVE_ARGS 8,1

	movq  %rax,ORIG_RAX-ARGOFFSET(%rsp) 

	movq  %rcx,RIP-ARGOFFSET(%rsp)
@@ -246,7 +261,7 @@ ret_from_sys_call: 
sysret_check:		

	LOCKDEP_SYS_EXIT

	GET_THREAD_INFO(%rcx)

	cli

	DISABLE_INTERRUPTS(CLBR_NONE)

	TRACE_IRQS_OFF

	movl threadinfo_flags(%rcx),%edx

	andl %edi,%edx
@@ -260,9 +275,7 @@ sysret_check: 
	CFI_REGISTER	rip,rcx

	RESTORE_ARGS 0,-ARG_SKIP,1

	/*CFI_REGISTER	rflags,r11*/

	movq	%gs:pda_oldrsp,%rsp

	swapgs

	sysretq

	ENABLE_INTERRUPTS_SYSCALL_RET



	CFI_RESTORE_STATE

	/* Handle reschedules */
@@ -271,7 +284,7 @@ sysret_careful: 
	bt $TIF_NEED_RESCHED,%edx

	jnc sysret_signal

	TRACE_IRQS_ON

	sti

	ENABLE_INTERRUPTS(CLBR_NONE)

	pushq %rdi

	CFI_ADJUST_CFA_OFFSET 8

	call schedule
@@ -282,7 +295,7 @@ sysret_careful: 
	/* Handle a signal */ 

sysret_signal:

	TRACE_IRQS_ON

	sti

	ENABLE_INTERRUPTS(CLBR_NONE)

	testl $_TIF_DO_NOTIFY_MASK,%edx

	jz    1f
@@ -295,7 +308,7 @@ sysret_signal: 
1:	movl $_TIF_NEED_RESCHED,%edi

	/* Use IRET because user could have changed frame. This

	   works because ptregscall_common has called FIXUP_TOP_OF_STACK. */

	cli

	DISABLE_INTERRUPTS(CLBR_NONE)

	TRACE_IRQS_OFF

	jmp int_with_check
@@ -327,7 +340,7 @@ tracesys: 
 */

	.globl int_ret_from_sys_call

int_ret_from_sys_call:

	cli

	DISABLE_INTERRUPTS(CLBR_NONE)

	TRACE_IRQS_OFF

	testl $3,CS-ARGOFFSET(%rsp)

	je retint_restore_args
@@ -349,20 +362,20 @@ int_careful: 
	bt $TIF_NEED_RESCHED,%edx

	jnc  int_very_careful

	TRACE_IRQS_ON

	sti

	ENABLE_INTERRUPTS(CLBR_NONE)

	pushq %rdi

	CFI_ADJUST_CFA_OFFSET 8

	call schedule

	popq %rdi

	CFI_ADJUST_CFA_OFFSET -8

	cli

	DISABLE_INTERRUPTS(CLBR_NONE)

	TRACE_IRQS_OFF

	jmp int_with_check



	/* handle signals and tracing -- both require a full stack frame */

int_very_careful:

	TRACE_IRQS_ON

	sti

	ENABLE_INTERRUPTS(CLBR_NONE)

	SAVE_REST

	/* Check for syscall exit trace */	

	testl $(_TIF_SYSCALL_TRACE|_TIF_SYSCALL_AUDIT|_TIF_SINGLESTEP),%edx
@@ -385,7 +398,7 @@ int_signal: 
1:	movl $_TIF_NEED_RESCHED,%edi	

int_restore_rest:

	RESTORE_REST

	cli

	DISABLE_INTERRUPTS(CLBR_NONE)

	TRACE_IRQS_OFF

	jmp int_with_check

	CFI_ENDPROC
@@ -506,7 +519,7 @@ END(stub_rt_sigreturn) 
	CFI_DEF_CFA_REGISTER	rbp

	testl $3,CS(%rdi)

	je 1f

	swapgs	

	SWAPGS

	/* irqcount is used to check if a CPU is already on an interrupt

	   stack or not. While this is essentially redundant with preempt_count

	   it is a little cheaper to use a separate counter in the PDA
@@ -527,7 +540,7 @@ ENTRY(common_interrupt) 
	interrupt do_IRQ

	/* 0(%rsp): oldrsp-ARGOFFSET */

ret_from_intr:

	cli	

	DISABLE_INTERRUPTS(CLBR_NONE)

	TRACE_IRQS_OFF

	decl %gs:pda_irqcount

	leaveq
@@ -556,13 +569,13 @@ retint_swapgs: /* return to user-space */ 
	/*

	 * The iretq could re-enable interrupts:

	 */

	cli

	DISABLE_INTERRUPTS(CLBR_ANY)

	TRACE_IRQS_IRETQ

	swapgs 

	SWAPGS

	jmp restore_args



retint_restore_args:	/* return to kernel space */

	cli

	DISABLE_INTERRUPTS(CLBR_ANY)

	/*

	 * The iretq could re-enable interrupts:

	 */
@@ -570,10 +583,14 @@ retint_restore_args: /* return to kernel space */ 
restore_args:

	RESTORE_ARGS 0,8,0						

iret_label:	

#ifdef CONFIG_PARAVIRT

	INTERRUPT_RETURN

#endif

ENTRY(native_iret)

	iretq



	.section __ex_table,"a"

	.quad iret_label,bad_iret	

	.quad native_iret, bad_iret

	.previous

	.section .fixup,"ax"

	/* force a signal here? this matches i386 behaviour */
@@ -581,7 +598,7 @@ iret_label: 
bad_iret:

	movq $11,%rdi	/* SIGSEGV */

	TRACE_IRQS_ON

	sti

	ENABLE_INTERRUPTS(CLBR_ANY | ~(CLBR_RDI))

	jmp do_exit

	.previous
@@ -591,14 +608,14 @@ retint_careful: 
	bt    $TIF_NEED_RESCHED,%edx

	jnc   retint_signal

	TRACE_IRQS_ON

	sti

	ENABLE_INTERRUPTS(CLBR_NONE)

	pushq %rdi

	CFI_ADJUST_CFA_OFFSET	8

	call  schedule

	popq %rdi		

	CFI_ADJUST_CFA_OFFSET	-8

	GET_THREAD_INFO(%rcx)

	cli

	DISABLE_INTERRUPTS(CLBR_NONE)

	TRACE_IRQS_OFF

	jmp retint_check
@@ -606,14 +623,14 @@ retint_signal: 
	testl $_TIF_DO_NOTIFY_MASK,%edx

	jz    retint_swapgs

	TRACE_IRQS_ON

	sti

	ENABLE_INTERRUPTS(CLBR_NONE)

	SAVE_REST

	movq $-1,ORIG_RAX(%rsp) 			

	xorl %esi,%esi		# oldset

	movq %rsp,%rdi		# &pt_regs

	call do_notify_resume

	RESTORE_REST

	cli

	DISABLE_INTERRUPTS(CLBR_NONE)

	TRACE_IRQS_OFF

	movl $_TIF_NEED_RESCHED,%edi

	GET_THREAD_INFO(%rcx)
@@ -731,7 +748,7 @@ END(spurious_interrupt) 
	rdmsr

	testl %edx,%edx

	js    1f

	swapgs

	SWAPGS

	xorl  %ebx,%ebx

1:

	.if \ist
@@ -747,7 +764,7 @@ END(spurious_interrupt) 
	.if \ist

	addq	$EXCEPTION_STKSZ, per_cpu__init_tss + TSS_ist + (\ist - 1) * 8(%rbp)

	.endif

	cli

	DISABLE_INTERRUPTS(CLBR_NONE)

	.if \irqtrace

	TRACE_IRQS_OFF

	.endif
@@ -776,10 +793,10 @@ paranoid_swapgs\trace: 
	.if \trace

	TRACE_IRQS_IRETQ 0

	.endif

	swapgs

	SWAPGS_UNSAFE_STACK

paranoid_restore\trace:

	RESTORE_ALL 8

	iretq

	INTERRUPT_RETURN

paranoid_userspace\trace:

	GET_THREAD_INFO(%rcx)

	movl threadinfo_flags(%rcx),%ebx
@@ -794,11 +811,11 @@ paranoid_userspace\trace: 
	.if \trace

	TRACE_IRQS_ON

	.endif

	sti

	ENABLE_INTERRUPTS(CLBR_NONE)

	xorl %esi,%esi 			/* arg2: oldset */

	movq %rsp,%rdi 			/* arg1: &pt_regs */

	call do_notify_resume

	cli

	DISABLE_INTERRUPTS(CLBR_NONE)

	.if \trace

	TRACE_IRQS_OFF

	.endif
@@ -807,9 +824,9 @@ paranoid_schedule\trace: 
	.if \trace

	TRACE_IRQS_ON

	.endif

	sti

	ENABLE_INTERRUPTS(CLBR_ANY)

	call schedule

	cli

	DISABLE_INTERRUPTS(CLBR_ANY)

	.if \trace

	TRACE_IRQS_OFF

	.endif
@@ -862,7 +879,7 @@ KPROBE_ENTRY(error_entry) 
	testl $3,CS(%rsp)

	je  error_kernelspace

error_swapgs:	

	swapgs

	SWAPGS

error_sti:	

	movq %rdi,RDI(%rsp) 	

	CFI_REL_OFFSET	rdi,RDI
@@ -874,7 +891,7 @@ error_sti: 
error_exit:

	movl %ebx,%eax

	RESTORE_REST

	cli

	DISABLE_INTERRUPTS(CLBR_NONE)

	TRACE_IRQS_OFF

	GET_THREAD_INFO(%rcx)	

	testl %eax,%eax
@@ -911,12 +928,12 @@ ENTRY(load_gs_index) 
	CFI_STARTPROC

	pushf

	CFI_ADJUST_CFA_OFFSET 8

	cli

        swapgs

	DISABLE_INTERRUPTS(CLBR_ANY | ~(CLBR_RDI))

        SWAPGS

gs_change:     

        movl %edi,%gs   

2:	mfence		/* workaround */

	swapgs

	SWAPGS

        popf

	CFI_ADJUST_CFA_OFFSET -8

        ret
@@ -930,7 +947,7 @@ ENDPROC(load_gs_index) 
        .section .fixup,"ax"

	/* running with kernelgs */

bad_gs: 

	swapgs			/* switch back to user gs */

	SWAPGS			/* switch back to user gs */

	xorl %eax,%eax

        movl %eax,%gs

        jmp  2b

CRA Git | Maintained and supported by SUSTech CRA and CCSE