Commit 5f409e20 authored by Rik van Riel's avatar Rik van Riel Committed by Borislav Petkov
Browse files

x86/fpu: Defer FPU state load until return to userspace 



Defer loading of FPU state until return to userspace. This gives
the kernel the potential to skip loading FPU state for tasks that
stay in kernel mode, or for tasks that end up with repeated
invocations of kernel_fpu_begin() & kernel_fpu_end().

The fpregs_lock/unlock() section ensures that the registers remain
unchanged. Otherwise a context switch or a bottom half could save the
registers to its FPU context and the processor's FPU registers would
became random if modified at the same time.

KVM swaps the host/guest registers on entry/exit path. This flow has
been kept as is. First it ensures that the registers are loaded and then
saves the current (host) state before it loads the guest's registers. The
swap is done at the very end with disabled interrupts so it should not
change anymore before theg guest is entered. The read/save version seems
to be cheaper compared to memcpy() in a micro benchmark.

Each thread gets TIF_NEED_FPU_LOAD set as part of fork() / fpu__copy().
For kernel threads, this flag gets never cleared which avoids saving /
restoring the FPU state for kernel threads and during in-kernel usage of
the FPU registers.

 [
   bp: Correct and update commit message and fix checkpatch warnings.
   s/register/registers/ where it is used in plural.
   minor comment corrections.
   remove unused trace_x86_fpu_activate_state() TP.
 ]

Signed-off-by: default avatarRik van Riel <riel@surriel.com>
Signed-off-by: default avatarSebastian Andrzej Siewior <bigeasy@linutronix.de>
Signed-off-by: default avatarBorislav Petkov <bp@suse.de>
Reviewed-by: default avatarDave Hansen <dave.hansen@intel.com>
Reviewed-by: default avatarThomas Gleixner <tglx@linutronix.de>
Cc: Andy Lutomirski <luto@kernel.org>
Cc: Aubrey Li <aubrey.li@intel.com>
Cc: Babu Moger <Babu.Moger@amd.com>
Cc: "Chang S. Bae" <chang.seok.bae@intel.com>
Cc: Dmitry Safonov <dima@arista.com>
Cc: "H. Peter Anvin" <hpa@zytor.com>
Cc: Ingo Molnar <mingo@redhat.com>
Cc: Jann Horn <jannh@google.com>
Cc: "Jason A. Donenfeld" <Jason@zx2c4.com>
Cc: Joerg Roedel <jroedel@suse.de>
Cc: Konrad Rzeszutek Wilk <konrad.wilk@oracle.com>
Cc: kvm ML <kvm@vger.kernel.org>
Cc: Nicolai Stange <nstange@suse.de>
Cc: Paolo Bonzini <pbonzini@redhat.com>
Cc: "Radim Krčmář" <rkrcmar@redhat.com>
Cc: Tim Chen <tim.c.chen@linux.intel.com>
Cc: Waiman Long <longman@redhat.com>
Cc: x86-ml <x86@kernel.org>
Cc: Yi Wang <wang.yi59@zte.com.cn>
Link: https://lkml.kernel.org/r/20190403164156.19645-24-bigeasy@linutronix.de
parent c2ff9e9a

arch/x86/entry/common.c

+9 −1
Original line number Diff line number Diff line
@@ -25,12 +25,13 @@ 
#include <linux/uprobes.h>

#include <linux/livepatch.h>

#include <linux/syscalls.h>

#include <linux/uaccess.h>



#include <asm/desc.h>

#include <asm/traps.h>

#include <asm/vdso.h>

#include <linux/uaccess.h>

#include <asm/cpufeature.h>

#include <asm/fpu/api.h>



#define CREATE_TRACE_POINTS

#include <trace/events/syscalls.h>
@@ -196,6 +197,13 @@ __visible inline void prepare_exit_to_usermode(struct pt_regs *regs) 
	if (unlikely(cached_flags & EXIT_TO_USERMODE_LOOP_FLAGS))

		exit_to_usermode_loop(regs, cached_flags);



	/* Reload ti->flags; we may have rescheduled above. */

	cached_flags = READ_ONCE(ti->flags);



	fpregs_assert_state_consistent();

	if (unlikely(cached_flags & _TIF_NEED_FPU_LOAD))

		switch_fpu_return();



#ifdef CONFIG_COMPAT

	/*

	 * Compat syscalls set TS_COMPAT.  Make sure we clear it before

arch/x86/include/asm/fpu/api.h

+21 −1
Original line number Diff line number Diff line
@@ -10,7 +10,7 @@ 


#ifndef _ASM_X86_FPU_API_H

#define _ASM_X86_FPU_API_H

#include <linux/preempt.h>

#include <linux/bottom_half.h>



/*

 * Use kernel_fpu_begin/end() if you intend to use FPU in kernel context. It
@@ -22,17 +22,37 @@ 
extern void kernel_fpu_begin(void);

extern void kernel_fpu_end(void);

extern bool irq_fpu_usable(void);

extern void fpregs_mark_activate(void);



/*

 * Use fpregs_lock() while editing CPU's FPU registers or fpu->state.

 * A context switch will (and softirq might) save CPU's FPU registers to

 * fpu->state and set TIF_NEED_FPU_LOAD leaving CPU's FPU registers in

 * a random state.

 */

static inline void fpregs_lock(void)

{

	preempt_disable();

	local_bh_disable();

}



static inline void fpregs_unlock(void)

{

	local_bh_enable();

	preempt_enable();

}



#ifdef CONFIG_X86_DEBUG_FPU

extern void fpregs_assert_state_consistent(void);

#else

static inline void fpregs_assert_state_consistent(void) { }

#endif



/*

 * Load the task FPU state before returning to userspace.

 */

extern void switch_fpu_return(void);



/*

 * Query the presence of one or more xfeatures. Works on any legacy CPU as well.

 *

arch/x86/include/asm/fpu/internal.h

+17 −10
Original line number Diff line number Diff line
@@ -30,7 +30,7 @@ extern void fpu__prepare_write(struct fpu *fpu); 
extern void fpu__save(struct fpu *fpu);

extern int  fpu__restore_sig(void __user *buf, int ia32_frame);

extern void fpu__drop(struct fpu *fpu);

extern int  fpu__copy(struct fpu *dst_fpu, struct fpu *src_fpu);

extern int  fpu__copy(struct task_struct *dst, struct task_struct *src);

extern void fpu__clear(struct fpu *fpu);

extern int  fpu__exception_code(struct fpu *fpu, int trap_nr);

extern int  dump_fpu(struct pt_regs *ptregs, struct user_i387_struct *fpstate);
@@ -531,13 +531,20 @@ static inline void fpregs_activate(struct fpu *fpu) 
/*

 * Internal helper, do not use directly. Use switch_fpu_return() instead.

 */

static inline void __fpregs_load_activate(struct fpu *fpu, int cpu)

static inline void __fpregs_load_activate(void)

{

	struct fpu *fpu = &current->thread.fpu;

	int cpu = smp_processor_id();



	if (WARN_ON_ONCE(current->mm == NULL))

		return;



	if (!fpregs_state_valid(fpu, cpu)) {

		if (current->mm)

		copy_kernel_to_fpregs(&fpu->state);

		fpregs_activate(fpu);

		fpu->last_cpu = cpu;

	}

	clear_thread_flag(TIF_NEED_FPU_LOAD);

}



/*
@@ -548,8 +555,8 @@ static inline void __fpregs_load_activate(struct fpu *fpu, int cpu) 
 *  - switch_fpu_prepare() saves the old state.

 *    This is done within the context of the old process.

 *

 *  - switch_fpu_finish() restores the new state as

 *    necessary.

 *  - switch_fpu_finish() sets TIF_NEED_FPU_LOAD; the floating point state

 *    will get loaded on return to userspace, or when the kernel needs it.

 *

 * If TIF_NEED_FPU_LOAD is cleared then the CPU's FPU registers

 * are saved in the current thread's FPU register state.
@@ -581,10 +588,10 @@ switch_fpu_prepare(struct fpu *old_fpu, int cpu) 
 */



/*

 * Set up the userspace FPU context for the new task, if the task

 * has used the FPU.

 * Load PKRU from the FPU context if available. Delay loading of the

 * complete FPU state until the return to userland.

 */

static inline void switch_fpu_finish(struct fpu *new_fpu, int cpu)

static inline void switch_fpu_finish(struct fpu *new_fpu)

{

	u32 pkru_val = init_pkru_value;

	struct pkru_state *pk;
@@ -592,7 +599,7 @@ static inline void switch_fpu_finish(struct fpu *new_fpu, int cpu) 
	if (!static_cpu_has(X86_FEATURE_FPU))

		return;



	__fpregs_load_activate(new_fpu, cpu);

	set_thread_flag(TIF_NEED_FPU_LOAD);



	if (!cpu_feature_enabled(X86_FEATURE_OSPKE))

		return;

arch/x86/include/asm/trace/fpu.h

+4 −6
Original line number Diff line number Diff line
@@ -13,19 +13,22 @@ DECLARE_EVENT_CLASS(x86_fpu, 


	TP_STRUCT__entry(

		__field(struct fpu *, fpu)

		__field(bool, load_fpu)

		__field(u64, xfeatures)

		__field(u64, xcomp_bv)

		),



	TP_fast_assign(

		__entry->fpu		= fpu;

		__entry->load_fpu	= test_thread_flag(TIF_NEED_FPU_LOAD);

		if (boot_cpu_has(X86_FEATURE_OSXSAVE)) {

			__entry->xfeatures = fpu->state.xsave.header.xfeatures;

			__entry->xcomp_bv  = fpu->state.xsave.header.xcomp_bv;

		}

	),

	TP_printk("x86/fpu: %p xfeatures: %llx xcomp_bv: %llx",

	TP_printk("x86/fpu: %p load: %d xfeatures: %llx xcomp_bv: %llx",

			__entry->fpu,

			__entry->load_fpu,

			__entry->xfeatures,

			__entry->xcomp_bv

	)
@@ -61,11 +64,6 @@ DEFINE_EVENT(x86_fpu, x86_fpu_regs_deactivated, 
	TP_ARGS(fpu)

);



DEFINE_EVENT(x86_fpu, x86_fpu_activate_state,

	TP_PROTO(struct fpu *fpu),

	TP_ARGS(fpu)

);



DEFINE_EVENT(x86_fpu, x86_fpu_init_state,

	TP_PROTO(struct fpu *fpu),

	TP_ARGS(fpu)

arch/x86/kernel/fpu/core.c

+80 −26
Original line number Diff line number Diff line
@@ -102,23 +102,20 @@ static void __kernel_fpu_begin(void) 
	kernel_fpu_disable();



	if (current->mm) {

		if (!test_thread_flag(TIF_NEED_FPU_LOAD)) {

			set_thread_flag(TIF_NEED_FPU_LOAD);

			/*

			 * Ignore return value -- we don't care if reg state

			 * is clobbered.

			 */

			copy_fpregs_to_fpstate(fpu);

	} else {

		__cpu_invalidate_fpregs_state();

		}

	}

	__cpu_invalidate_fpregs_state();

}



static void __kernel_fpu_end(void)

{

	struct fpu *fpu = &current->thread.fpu;



	if (current->mm)

		copy_kernel_to_fpregs(&fpu->state);



	kernel_fpu_enable();

}
@@ -145,14 +142,17 @@ void fpu__save(struct fpu *fpu) 
{

	WARN_ON_FPU(fpu != &current->thread.fpu);



	preempt_disable();

	fpregs_lock();

	trace_x86_fpu_before_save(fpu);



	if (!copy_fpregs_to_fpstate(fpu))

	if (!test_thread_flag(TIF_NEED_FPU_LOAD)) {

		if (!copy_fpregs_to_fpstate(fpu)) {

			copy_kernel_to_fpregs(&fpu->state);

		}

	}



	trace_x86_fpu_after_save(fpu);

	preempt_enable();

	fpregs_unlock();

}

EXPORT_SYMBOL_GPL(fpu__save);
@@ -185,8 +185,11 @@ void fpstate_init(union fpregs_state *state) 
}

EXPORT_SYMBOL_GPL(fpstate_init);



int fpu__copy(struct fpu *dst_fpu, struct fpu *src_fpu)

int fpu__copy(struct task_struct *dst, struct task_struct *src)

{

	struct fpu *dst_fpu = &dst->thread.fpu;

	struct fpu *src_fpu = &src->thread.fpu;



	dst_fpu->last_cpu = -1;



	if (!static_cpu_has(X86_FEATURE_FPU))
@@ -201,16 +204,23 @@ int fpu__copy(struct fpu *dst_fpu, struct fpu *src_fpu) 
	memset(&dst_fpu->state.xsave, 0, fpu_kernel_xstate_size);



	/*

	 * Save current FPU registers directly into the child

	 * FPU context, without any memory-to-memory copying.

	 * If the FPU registers are not current just memcpy() the state.

	 * Otherwise save current FPU registers directly into the child's FPU

	 * context, without any memory-to-memory copying.

	 *

	 * ( The function 'fails' in the FNSAVE case, which destroys

	 *   register contents so we have to copy them back. )

	 *   register contents so we have to load them back. )

	 */

	if (!copy_fpregs_to_fpstate(dst_fpu)) {

		memcpy(&src_fpu->state, &dst_fpu->state, fpu_kernel_xstate_size);

		copy_kernel_to_fpregs(&src_fpu->state);

	}

	fpregs_lock();

	if (test_thread_flag(TIF_NEED_FPU_LOAD))

		memcpy(&dst_fpu->state, &src_fpu->state, fpu_kernel_xstate_size);



	else if (!copy_fpregs_to_fpstate(dst_fpu))

		copy_kernel_to_fpregs(&dst_fpu->state);



	fpregs_unlock();



	set_tsk_thread_flag(dst, TIF_NEED_FPU_LOAD);



	trace_x86_fpu_copy_src(src_fpu);

	trace_x86_fpu_copy_dst(dst_fpu);
@@ -226,10 +236,9 @@ static void fpu__initialize(struct fpu *fpu) 
{

	WARN_ON_FPU(fpu != &current->thread.fpu);



	set_thread_flag(TIF_NEED_FPU_LOAD);

	fpstate_init(&fpu->state);

	trace_x86_fpu_init_state(fpu);



	trace_x86_fpu_activate_state(fpu);

}



/*
@@ -308,6 +317,8 @@ void fpu__drop(struct fpu *fpu) 
 */

static inline void copy_init_fpstate_to_fpregs(void)

{

	fpregs_lock();



	if (use_xsave())

		copy_kernel_to_xregs(&init_fpstate.xsave, -1);

	else if (static_cpu_has(X86_FEATURE_FXSR))
@@ -317,6 +328,9 @@ static inline void copy_init_fpstate_to_fpregs(void) 


	if (boot_cpu_has(X86_FEATURE_OSPKE))

		copy_init_pkru_to_fpregs();



	fpregs_mark_activate();

	fpregs_unlock();

}



/*
@@ -339,6 +353,46 @@ void fpu__clear(struct fpu *fpu) 
		copy_init_fpstate_to_fpregs();

}



/*

 * Load FPU context before returning to userspace.

 */

void switch_fpu_return(void)

{

	if (!static_cpu_has(X86_FEATURE_FPU))

		return;



	__fpregs_load_activate();

}

EXPORT_SYMBOL_GPL(switch_fpu_return);



#ifdef CONFIG_X86_DEBUG_FPU

/*

 * If current FPU state according to its tracking (loaded FPU context on this

 * CPU) is not valid then we must have TIF_NEED_FPU_LOAD set so the context is

 * loaded on return to userland.

 */

void fpregs_assert_state_consistent(void)

{

	struct fpu *fpu = &current->thread.fpu;



	if (test_thread_flag(TIF_NEED_FPU_LOAD))

		return;



	WARN_ON_FPU(!fpregs_state_valid(fpu, smp_processor_id()));

}

EXPORT_SYMBOL_GPL(fpregs_assert_state_consistent);

#endif



void fpregs_mark_activate(void)

{

	struct fpu *fpu = &current->thread.fpu;



	fpregs_activate(fpu);

	fpu->last_cpu = smp_processor_id();

	clear_thread_flag(TIF_NEED_FPU_LOAD);

}

EXPORT_SYMBOL_GPL(fpregs_mark_activate);



/*

 * x87 math exception handling:

 */

CRA Git | Maintained and supported by SUSTech CRA and CCSE