Commit a063bf24 authored by Kan Liang's avatar Kan Liang Committed by Peter Zijlstra
Browse files

x86/fpu: Use proper mask to replace full instruction mask 



When saving xstate to a kernel/user XSAVE area with the XSAVE family of
instructions, the current code applies the 'full' instruction mask (-1),
which tries to XSAVE all possible features. This method relies on
hardware to trim 'all possible' down to what is enabled in the
hardware. The code works well for now. However, there will be a
problem, if some features are enabled in hardware, but are not suitable
to be saved into all kernel XSAVE buffers, like task->fpu, due to
performance consideration.

One such example is the Last Branch Records (LBR) state. The LBR state
only contains valuable information when LBR is explicitly enabled by
the perf subsystem, and the size of an LBR state is large (808 bytes
for now). To avoid both CPU overhead and space overhead at each context
switch, the LBR state should not be saved into task->fpu like other
state components. It should be saved/restored on demand when LBR is
enabled in the perf subsystem. Current copy_xregs_to_* will trigger a
buffer overflow for such cases.

Three sites use the '-1' instruction mask which must be updated.

Two are saving/restoring the xstate to/from a kernel-allocated XSAVE
buffer and can use 'xfeatures_mask_all', which will save/restore all of
the features present in a normal task FPU buffer.

The last one saves the register state directly to a user buffer. It
could
also use 'xfeatures_mask_all'. Just as it was with the '-1' argument,
any supervisor states in the mask will be filtered out by the hardware
and not saved to the buffer.  But, to be more explicit about what is
expected to be saved, use xfeatures_mask_user() for the instruction
mask.

KVM includes the header file fpu/internal.h. To avoid 'undefined
xfeatures_mask_all' compiling issue, move copy_fpregs_to_fpstate() to
fpu/core.c and export it, because:
- The xfeatures_mask_all is indirectly used via copy_fpregs_to_fpstate()
  by KVM. The function which is directly used by other modules should be
  exported.
- The copy_fpregs_to_fpstate() is a function, while xfeatures_mask_all
  is a variable for the "internal" FPU state. It's safer to export a
  function than a variable, which may be implicitly changed by others.
- The copy_fpregs_to_fpstate() is a big function with many checks. The
  removal of the inline keyword should not impact the performance.

Signed-off-by: default avatarKan Liang <kan.liang@linux.intel.com>
Signed-off-by: default avatarPeter Zijlstra (Intel) <peterz@infradead.org>
Reviewed-by: default avatarDave Hansen <dave.hansen@intel.com>
Link: https://lkml.kernel.org/r/1593780569-62993-20-git-send-email-kan.liang@linux.intel.com
parent 5a09928d

arch/x86/include/asm/fpu/internal.h

+7 −40
Original line number Diff line number Diff line
@@ -274,7 +274,7 @@ static inline void copy_fxregs_to_kernel(struct fpu *fpu) 
 */

static inline void copy_xregs_to_kernel_booting(struct xregs_state *xstate)

{

	u64 mask = -1;

	u64 mask = xfeatures_mask_all;

	u32 lmask = mask;

	u32 hmask = mask >> 32;

	int err;
@@ -320,7 +320,7 @@ static inline void copy_kernel_to_xregs_booting(struct xregs_state *xstate) 
 */

static inline void copy_xregs_to_kernel(struct xregs_state *xstate)

{

	u64 mask = -1;

	u64 mask = xfeatures_mask_all;

	u32 lmask = mask;

	u32 hmask = mask >> 32;

	int err;
@@ -356,6 +356,9 @@ static inline void copy_kernel_to_xregs(struct xregs_state *xstate, u64 mask) 
 */

static inline int copy_xregs_to_user(struct xregs_state __user *buf)

{

	u64 mask = xfeatures_mask_user();

	u32 lmask = mask;

	u32 hmask = mask >> 32;

	int err;



	/*
@@ -367,7 +370,7 @@ static inline int copy_xregs_to_user(struct xregs_state __user *buf) 
		return -EFAULT;



	stac();

	XSTATE_OP(XSAVE, buf, -1, -1, err);

	XSTATE_OP(XSAVE, buf, lmask, hmask, err);

	clac();



	return err;
@@ -408,43 +411,7 @@ static inline int copy_kernel_to_xregs_err(struct xregs_state *xstate, u64 mask) 
	return err;

}



/*

 * These must be called with preempt disabled. Returns

 * 'true' if the FPU state is still intact and we can

 * keep registers active.

 *

 * The legacy FNSAVE instruction cleared all FPU state

 * unconditionally, so registers are essentially destroyed.

 * Modern FPU state can be kept in registers, if there are

 * no pending FP exceptions.

 */

static inline int copy_fpregs_to_fpstate(struct fpu *fpu)

{

	if (likely(use_xsave())) {

		copy_xregs_to_kernel(&fpu->state.xsave);



		/*

		 * AVX512 state is tracked here because its use is

		 * known to slow the max clock speed of the core.

		 */

		if (fpu->state.xsave.header.xfeatures & XFEATURE_MASK_AVX512)

			fpu->avx512_timestamp = jiffies;

		return 1;

	}



	if (likely(use_fxsr())) {

		copy_fxregs_to_kernel(fpu);

		return 1;

	}



	/*

	 * Legacy FPU register saving, FNSAVE always clears FPU registers,

	 * so we have to mark them inactive:

	 */

	asm volatile("fnsave %[fp]; fwait" : [fp] "=m" (fpu->state.fsave));



	return 0;

}

extern int copy_fpregs_to_fpstate(struct fpu *fpu);



static inline void __copy_kernel_to_fpregs(union fpregs_state *fpstate, u64 mask)

{

arch/x86/kernel/fpu/core.c

+39 −0
Original line number Diff line number Diff line
@@ -82,6 +82,45 @@ bool irq_fpu_usable(void) 
}

EXPORT_SYMBOL(irq_fpu_usable);



/*

 * These must be called with preempt disabled. Returns

 * 'true' if the FPU state is still intact and we can

 * keep registers active.

 *

 * The legacy FNSAVE instruction cleared all FPU state

 * unconditionally, so registers are essentially destroyed.

 * Modern FPU state can be kept in registers, if there are

 * no pending FP exceptions.

 */

int copy_fpregs_to_fpstate(struct fpu *fpu)

{

	if (likely(use_xsave())) {

		copy_xregs_to_kernel(&fpu->state.xsave);



		/*

		 * AVX512 state is tracked here because its use is

		 * known to slow the max clock speed of the core.

		 */

		if (fpu->state.xsave.header.xfeatures & XFEATURE_MASK_AVX512)

			fpu->avx512_timestamp = jiffies;

		return 1;

	}



	if (likely(use_fxsr())) {

		copy_fxregs_to_kernel(fpu);

		return 1;

	}



	/*

	 * Legacy FPU register saving, FNSAVE always clears FPU registers,

	 * so we have to mark them inactive:

	 */

	asm volatile("fnsave %[fp]; fwait" : [fp] "=m" (fpu->state.fsave));



	return 0;

}

EXPORT_SYMBOL(copy_fpregs_to_fpstate);



void kernel_fpu_begin(void)

{

	preempt_disable();

CRA Git | Maintained and supported by SUSTech CRA and CCSE