Commit 9104a18d authored by Jeremy Fitzhardinge's avatar Jeremy Fitzhardinge Committed by H. Peter Anvin
Browse files

x86/paravirt: selectively save/restore regs around pvops calls 



Impact: Optimization

Each asm paravirt-ops call says what registers are available for
clobbering.  This patch makes use of this to selectively save/restore
registers around each pvops call.  In many cases this significantly
shrinks code size.

Signed-off-by: default avatarJeremy Fitzhardinge <jeremy.fitzhardinge@citrix.com>
Signed-off-by: default avatarH. Peter Anvin <hpa@zytor.com>
parent b8aa287f

arch/x86/include/asm/paravirt.h

+65 −35
Original line number Diff line number Diff line
@@ -12,19 +12,29 @@ 
#define CLBR_EAX  (1 << 0)

#define CLBR_ECX  (1 << 1)

#define CLBR_EDX  (1 << 2)

#define CLBR_EDI  (1 << 3)



#ifdef CONFIG_X86_64

#define CLBR_RSI  (1 << 3)

#define CLBR_RDI  (1 << 4)

#ifdef CONFIG_X86_32

/* CLBR_ANY should match all regs platform has. For i386, that's just it */

#define CLBR_ANY  ((1 << 4) - 1)

#else

#define CLBR_RAX  CLBR_EAX

#define CLBR_RCX  CLBR_ECX

#define CLBR_RDX  CLBR_EDX

#define CLBR_RDI  CLBR_EDI

#define CLBR_RSI  (1 << 4)

#define CLBR_R8   (1 << 5)

#define CLBR_R9   (1 << 6)

#define CLBR_R10  (1 << 7)

#define CLBR_R11  (1 << 8)

#define CLBR_ANY  ((1 << 9) - 1)



#define CLBR_ARG_REGS	(CLBR_RDI | CLBR_RSI | CLBR_RDX | \

			 CLBR_RCX | CLBR_R8 | CLBR_R9)

#define CLBR_RET_REG	(CLBR_RAX | CLBR_RDX)

#define CLBR_SCRATCH	(CLBR_R10 | CLBR_R11)



#include <asm/desc_defs.h>

#else

/* CLBR_ANY should match all regs platform has. For i386, that's just it */

#define CLBR_ANY  ((1 << 3) - 1)

#endif /* X86_64 */



#ifndef __ASSEMBLY__
@@ -1530,33 +1540,49 @@ static inline unsigned long __raw_local_irq_save(void) 
	.popsection





#define COND_PUSH(set, mask, reg)			\

	.if ((~set) & mask); push %reg; .endif

#define COND_POP(set, mask, reg)			\

	.if ((~set) & mask); pop %reg; .endif



#ifdef CONFIG_X86_64

#define PV_SAVE_REGS				\

	push %rax;				\

	push %rcx;				\

	push %rdx;				\

	push %rsi;				\

	push %rdi;				\

	push %r8;				\

	push %r9;				\

	push %r10;				\

	push %r11

#define PV_RESTORE_REGS				\

	pop %r11;				\

	pop %r10;				\

	pop %r9;				\

	pop %r8;				\

	pop %rdi;				\

	pop %rsi;				\

	pop %rdx;				\

	pop %rcx;				\

	pop %rax



#define PV_SAVE_REGS(set)			\

	COND_PUSH(set, CLBR_RAX, rax);		\

	COND_PUSH(set, CLBR_RCX, rcx);		\

	COND_PUSH(set, CLBR_RDX, rdx);		\

	COND_PUSH(set, CLBR_RSI, rsi);		\

	COND_PUSH(set, CLBR_RDI, rdi);		\

	COND_PUSH(set, CLBR_R8, r8);		\

	COND_PUSH(set, CLBR_R9, r9);		\

	COND_PUSH(set, CLBR_R10, r10);		\

	COND_PUSH(set, CLBR_R11, r11)

#define PV_RESTORE_REGS(set)			\

	COND_POP(set, CLBR_R11, r11);		\

	COND_POP(set, CLBR_R10, r10);		\

	COND_POP(set, CLBR_R9, r9);		\

	COND_POP(set, CLBR_R8, r8);		\

	COND_POP(set, CLBR_RDI, rdi);		\

	COND_POP(set, CLBR_RSI, rsi);		\

	COND_POP(set, CLBR_RDX, rdx);		\

	COND_POP(set, CLBR_RCX, rcx);		\

	COND_POP(set, CLBR_RAX, rax)



#define PARA_PATCH(struct, off)        ((PARAVIRT_PATCH_##struct + (off)) / 8)

#define PARA_SITE(ptype, clobbers, ops) _PVSITE(ptype, clobbers, ops, .quad, 8)

#define PARA_INDIRECT(addr)	*addr(%rip)

#else

#define PV_SAVE_REGS   pushl %eax; pushl %edi; pushl %ecx; pushl %edx

#define PV_RESTORE_REGS popl %edx; popl %ecx; popl %edi; popl %eax

#define PV_SAVE_REGS(set)			\

	COND_PUSH(set, CLBR_EAX, eax);		\

	COND_PUSH(set, CLBR_EDI, edi);		\

	COND_PUSH(set, CLBR_ECX, ecx);		\

	COND_PUSH(set, CLBR_EDX, edx)

#define PV_RESTORE_REGS(set)			\

	COND_POP(set, CLBR_EDX, edx);		\

	COND_POP(set, CLBR_ECX, ecx);		\

	COND_POP(set, CLBR_EDI, edi);		\

	COND_POP(set, CLBR_EAX, eax)



#define PARA_PATCH(struct, off)        ((PARAVIRT_PATCH_##struct + (off)) / 4)

#define PARA_SITE(ptype, clobbers, ops) _PVSITE(ptype, clobbers, ops, .long, 4)

#define PARA_INDIRECT(addr)	*%cs:addr
@@ -1568,15 +1594,15 @@ static inline unsigned long __raw_local_irq_save(void) 


#define DISABLE_INTERRUPTS(clobbers)					\

	PARA_SITE(PARA_PATCH(pv_irq_ops, PV_IRQ_irq_disable), clobbers, \

		  PV_SAVE_REGS;						\

		  PV_SAVE_REGS(clobbers);				\

		  call PARA_INDIRECT(pv_irq_ops+PV_IRQ_irq_disable);	\

		  PV_RESTORE_REGS;)			\

		  PV_RESTORE_REGS(clobbers);)



#define ENABLE_INTERRUPTS(clobbers)					\

	PARA_SITE(PARA_PATCH(pv_irq_ops, PV_IRQ_irq_enable), clobbers,	\

		  PV_SAVE_REGS;						\

		  PV_SAVE_REGS(clobbers);				\

		  call PARA_INDIRECT(pv_irq_ops+PV_IRQ_irq_enable);	\

		  PV_RESTORE_REGS;)

		  PV_RESTORE_REGS(clobbers);)



#define USERGS_SYSRET32							\

	PARA_SITE(PARA_PATCH(pv_cpu_ops, PV_CPU_usergs_sysret32),	\
@@ -1606,11 +1632,15 @@ static inline unsigned long __raw_local_irq_save(void) 
	PARA_SITE(PARA_PATCH(pv_cpu_ops, PV_CPU_swapgs), CLBR_NONE,	\

		  swapgs)



/*

 * Note: swapgs is very special, and in practise is either going to be

 * implemented with a single "swapgs" instruction or something very

 * special.  Either way, we don't need to save any registers for

 * it.

 */

#define SWAPGS								\

	PARA_SITE(PARA_PATCH(pv_cpu_ops, PV_CPU_swapgs), CLBR_NONE,	\

		  PV_SAVE_REGS;						\

		  call PARA_INDIRECT(pv_cpu_ops+PV_CPU_swapgs);		\

		  PV_RESTORE_REGS					\

		  call PARA_INDIRECT(pv_cpu_ops+PV_CPU_swapgs)		\

		 )



#define GET_CR2_INTO_RCX				\

CRA Git | Maintained and supported by SUSTech CRA and CCSE