Commit 7453f33b authored by Linus Torvalds's avatar Linus Torvalds
Browse files

Merge branch 'x86-xsave-for-linus' of git://git.kernel.org/pub/scm/linux/kernel/git/tip/tip 

Pull x86/xsave changes from Peter Anvin:
 "This is a patchset to support the XSAVES instruction required to
  support context switch of supervisor-only features in upcoming
  silicon.

  This patchset missed the 3.16 merge window, which is why it is based
  on 3.15-rc7"

* 'x86-xsave-for-linus' of git://git.kernel.org/pub/scm/linux/kernel/git/tip/tip:
  x86, xsave: Add forgotten inline annotation
  x86/xsaves: Clean up code in xstate offsets computation in xsave area
  x86/xsave: Make it clear that the XSAVE macros use (%edi)/(%rdi)
  Define kernel API to get address of each state in xsave area
  x86/xsaves: Enable xsaves/xrstors
  x86/xsaves: Call booting time xsaves and xrstors in setup_init_fpu_buf
  x86/xsaves: Save xstate to task's xsave area in __save_fpu during booting time
  x86/xsaves: Add xsaves and xrstors support for booting time
  x86/xsaves: Clear reserved bits in xsave header
  x86/xsaves: Use xsave/xrstor for saving and restoring user space context
  x86/xsaves: Use xsaves/xrstors for context switch
  x86/xsaves: Use xsaves/xrstors to save and restore xsave area
  x86/xsaves: Define a macro for handling xsave/xrstor instruction fault
  x86/xsaves: Define macros for xsave instructions
  x86/xsaves: Change compacted format xsave area header
  x86/alternative: Add alternative_input_2 to support alternative with two features and input
  x86/xsaves: Add a kernel parameter noxsaves to disable xsaves/xrstors
parents fd1cf905 d0f2dd18

Documentation/kernel-parameters.txt

+15 −0
Original line number Diff line number Diff line
@@ -2200,6 +2200,21 @@ bytes respectively. Such letter suffixes can also be entirely omitted. 
			and restore using xsave. The kernel will fallback to

			enabling legacy floating-point and sse state.



	noxsaveopt	[X86] Disables xsaveopt used in saving x86 extended

			register states. The kernel will fall back to use

			xsave to save the states. By using this parameter,

			performance of saving the states is degraded because

			xsave doesn't support modified optimization while

			xsaveopt supports it on xsaveopt enabled systems.



	noxsaves	[X86] Disables xsaves and xrstors used in saving and

			restoring x86 extended register state in compacted

			form of xsave area. The kernel will fall back to use

			xsaveopt and xrstor to save and restore the states

			in standard form of xsave area. By using this

			parameter, xsave area per process might occupy more

			memory on xsaves enabled systems.



	eagerfpu=	[X86]

			on	enable eager fpu restore

			off	disable eager fpu restore

arch/x86/include/asm/alternative.h

+14 −0
Original line number Diff line number Diff line
@@ -161,6 +161,20 @@ static inline int alternatives_text_reserved(void *start, void *end) 
	asm volatile (ALTERNATIVE(oldinstr, newinstr, feature)		\

		: : "i" (0), ## input)



/*

 * This is similar to alternative_input. But it has two features and

 * respective instructions.

 *

 * If CPU has feature2, newinstr2 is used.

 * Otherwise, if CPU has feature1, newinstr1 is used.

 * Otherwise, oldinstr is used.

 */

#define alternative_input_2(oldinstr, newinstr1, feature1, newinstr2,	     \

			   feature2, input...)				     \

	asm volatile(ALTERNATIVE_2(oldinstr, newinstr1, feature1,	     \

		newinstr2, feature2)					     \

		: : "i" (0), ## input)



/* Like alternative_input, but with a single output argument */

#define alternative_io(oldinstr, newinstr, feature, output, input...)	\

	asm volatile (ALTERNATIVE(oldinstr, newinstr, feature)		\

arch/x86/include/asm/fpu-internal.h

+6 −3
Original line number Diff line number Diff line
@@ -508,9 +508,12 @@ static inline void user_fpu_begin(void) 


static inline void __save_fpu(struct task_struct *tsk)

{

	if (use_xsave())

		xsave_state(&tsk->thread.fpu.state->xsave, -1);

	if (use_xsave()) {

		if (unlikely(system_state == SYSTEM_BOOTING))

			xsave_state_booting(&tsk->thread.fpu.state->xsave, -1);

		else

			xsave_state(&tsk->thread.fpu.state->xsave, -1);

	} else

		fpu_fxsave(&tsk->thread.fpu);

}

arch/x86/include/asm/processor.h

+2 −2
Original line number Diff line number Diff line
@@ -385,8 +385,8 @@ struct bndcsr_struct { 


struct xsave_hdr_struct {

	u64 xstate_bv;

	u64 reserved1[2];

	u64 reserved2[5];

	u64 xcomp_bv;

	u64 reserved[6];

} __attribute__((packed));



struct xsave_struct {

arch/x86/include/asm/xsave.h

+167 −56
Original line number Diff line number Diff line
@@ -52,24 +52,170 @@ extern void xsave_init(void); 
extern void update_regset_xstate_info(unsigned int size, u64 xstate_mask);

extern int init_fpu(struct task_struct *child);



static inline int fpu_xrstor_checking(struct xsave_struct *fx)

{

	int err;

/* These macros all use (%edi)/(%rdi) as the single memory argument. */

#define XSAVE		".byte " REX_PREFIX "0x0f,0xae,0x27"

#define XSAVEOPT	".byte " REX_PREFIX "0x0f,0xae,0x37"

#define XSAVES		".byte " REX_PREFIX "0x0f,0xc7,0x2f"

#define XRSTOR		".byte " REX_PREFIX "0x0f,0xae,0x2f"

#define XRSTORS		".byte " REX_PREFIX "0x0f,0xc7,0x1f"



	asm volatile("1: .byte " REX_PREFIX "0x0f,0xae,0x2f\n\t"

		     "2:\n"

		     ".section .fixup,\"ax\"\n"

		     "3:  movl $-1,%[err]\n"

		     "    jmp  2b\n"

		     ".previous\n"

		     _ASM_EXTABLE(1b, 3b)

#define xstate_fault	".section .fixup,\"ax\"\n"	\

			"3:  movl $-1,%[err]\n"		\

			"    jmp  2b\n"			\

			".previous\n"			\

			_ASM_EXTABLE(1b, 3b)		\

			: [err] "=r" (err)

		     : "D" (fx), "m" (*fx), "a" (-1), "d" (-1), "0" (0)



/*

 * This function is called only during boot time when x86 caps are not set

 * up and alternative can not be used yet.

 */

static inline int xsave_state_booting(struct xsave_struct *fx, u64 mask)

{

	u32 lmask = mask;

	u32 hmask = mask >> 32;

	int err = 0;



	WARN_ON(system_state != SYSTEM_BOOTING);



	if (boot_cpu_has(X86_FEATURE_XSAVES))

		asm volatile("1:"XSAVES"\n\t"

			"2:\n\t"

			: : "D" (fx), "m" (*fx), "a" (lmask), "d" (hmask)

			:   "memory");

	else

		asm volatile("1:"XSAVE"\n\t"

			"2:\n\t"

			: : "D" (fx), "m" (*fx), "a" (lmask), "d" (hmask)

			:   "memory");



	asm volatile(xstate_fault

		     : "0" (0)

		     : "memory");



	return err;

}



/*

 * This function is called only during boot time when x86 caps are not set

 * up and alternative can not be used yet.

 */

static inline int xrstor_state_booting(struct xsave_struct *fx, u64 mask)

{

	u32 lmask = mask;

	u32 hmask = mask >> 32;

	int err = 0;



	WARN_ON(system_state != SYSTEM_BOOTING);



	if (boot_cpu_has(X86_FEATURE_XSAVES))

		asm volatile("1:"XRSTORS"\n\t"

			"2:\n\t"

			: : "D" (fx), "m" (*fx), "a" (lmask), "d" (hmask)

			:   "memory");

	else

		asm volatile("1:"XRSTOR"\n\t"

			"2:\n\t"

			: : "D" (fx), "m" (*fx), "a" (lmask), "d" (hmask)

			:   "memory");



	asm volatile(xstate_fault

		     : "0" (0)

		     : "memory");



	return err;

}



/*

 * Save processor xstate to xsave area.

 */

static inline int xsave_state(struct xsave_struct *fx, u64 mask)

{

	u32 lmask = mask;

	u32 hmask = mask >> 32;

	int err = 0;



	/*

	 * If xsaves is enabled, xsaves replaces xsaveopt because

	 * it supports compact format and supervisor states in addition to

	 * modified optimization in xsaveopt.

	 *

	 * Otherwise, if xsaveopt is enabled, xsaveopt replaces xsave

	 * because xsaveopt supports modified optimization which is not

	 * supported by xsave.

	 *

	 * If none of xsaves and xsaveopt is enabled, use xsave.

	 */

	alternative_input_2(

		"1:"XSAVE,

		"1:"XSAVEOPT,

		X86_FEATURE_XSAVEOPT,

		"1:"XSAVES,

		X86_FEATURE_XSAVES,

		[fx] "D" (fx), "a" (lmask), "d" (hmask) :

		"memory");

	asm volatile("2:\n\t"

		     xstate_fault

		     : "0" (0)

		     : "memory");



	return err;

}



/*

 * Restore processor xstate from xsave area.

 */

static inline int xrstor_state(struct xsave_struct *fx, u64 mask)

{

	int err = 0;

	u32 lmask = mask;

	u32 hmask = mask >> 32;



	/*

	 * Use xrstors to restore context if it is enabled. xrstors supports

	 * compacted format of xsave area which is not supported by xrstor.

	 */

	alternative_input(

		"1: " XRSTOR,

		"1: " XRSTORS,

		X86_FEATURE_XSAVES,

		"D" (fx), "m" (*fx), "a" (lmask), "d" (hmask)

		: "memory");



	asm volatile("2:\n"

		     xstate_fault

		     : "0" (0)

		     : "memory");



	return err;

}



/*

 * Save xstate context for old process during context switch.

 */

static inline void fpu_xsave(struct fpu *fpu)

{

	xsave_state(&fpu->state->xsave, -1);

}



/*

 * Restore xstate context for new process during context switch.

 */

static inline int fpu_xrstor_checking(struct xsave_struct *fx)

{

	return xrstor_state(fx, -1);

}



/*

 * Save xstate to user space xsave area.

 *

 * We don't use modified optimization because xrstor/xrstors might track

 * a different application.

 *

 * We don't use compacted format xsave area for

 * backward compatibility for old applications which don't understand

 * compacted format of xsave area.

 */

static inline int xsave_user(struct xsave_struct __user *buf)

{

	int err;
@@ -83,69 +229,34 @@ static inline int xsave_user(struct xsave_struct __user *buf) 
		return -EFAULT;



	__asm__ __volatile__(ASM_STAC "\n"

			     "1: .byte " REX_PREFIX "0x0f,0xae,0x27\n"

			     "1:"XSAVE"\n"

			     "2: " ASM_CLAC "\n"

			     ".section .fixup,\"ax\"\n"

			     "3:  movl $-1,%[err]\n"

			     "    jmp  2b\n"

			     ".previous\n"

			     _ASM_EXTABLE(1b,3b)

			     : [err] "=r" (err)

			     xstate_fault

			     : "D" (buf), "a" (-1), "d" (-1), "0" (0)

			     : "memory");

	return err;

}



/*

 * Restore xstate from user space xsave area.

 */

static inline int xrestore_user(struct xsave_struct __user *buf, u64 mask)

{

	int err;

	int err = 0;

	struct xsave_struct *xstate = ((__force struct xsave_struct *)buf);

	u32 lmask = mask;

	u32 hmask = mask >> 32;



	__asm__ __volatile__(ASM_STAC "\n"

			     "1: .byte " REX_PREFIX "0x0f,0xae,0x2f\n"

			     "1:"XRSTOR"\n"

			     "2: " ASM_CLAC "\n"

			     ".section .fixup,\"ax\"\n"

			     "3:  movl $-1,%[err]\n"

			     "    jmp  2b\n"

			     ".previous\n"

			     _ASM_EXTABLE(1b,3b)

			     : [err] "=r" (err)

			     xstate_fault

			     : "D" (xstate), "a" (lmask), "d" (hmask), "0" (0)

			     : "memory");	/* memory required? */

	return err;

}



static inline void xrstor_state(struct xsave_struct *fx, u64 mask)

{

	u32 lmask = mask;

	u32 hmask = mask >> 32;



	asm volatile(".byte " REX_PREFIX "0x0f,0xae,0x2f\n\t"

		     : : "D" (fx), "m" (*fx), "a" (lmask), "d" (hmask)

		     :   "memory");

}



static inline void xsave_state(struct xsave_struct *fx, u64 mask)

{

	u32 lmask = mask;

	u32 hmask = mask >> 32;



	asm volatile(".byte " REX_PREFIX "0x0f,0xae,0x27\n\t"

		     : : "D" (fx), "m" (*fx), "a" (lmask), "d" (hmask)

		     :   "memory");

}

void *get_xsave_addr(struct xsave_struct *xsave, int xstate);

void setup_xstate_comp(void);



static inline void fpu_xsave(struct fpu *fpu)

{

	/* This, however, we can work around by forcing the compiler to select

	   an addressing mode that doesn't require extended registers. */

	alternative_input(

		".byte " REX_PREFIX "0x0f,0xae,0x27",

		".byte " REX_PREFIX "0x0f,0xae,0x37",

		X86_FEATURE_XSAVEOPT,

		[fx] "D" (&fpu->state->xsave), "a" (-1), "d" (-1) :

		"memory");

}

#endif

CRA Git | Maintained and supported by SUSTech CRA and CCSE