Commit bc696ca0 authored by Borislav Petkov's avatar Borislav Petkov Committed by Ingo Molnar
Browse files

x86/cpufeature: Replace the old static_cpu_has() with safe variant 



So the old one didn't work properly before alternatives had run.
And it was supposed to provide an optimized JMP because the
assumption was that the offset it is jumping to is within a
signed byte and thus a two-byte JMP.

So I did an x86_64 allyesconfig build and dumped all possible
sites where static_cpu_has() was used. The optimization amounted
to all in all 12(!) places where static_cpu_has() had generated
a 2-byte JMP. Which has saved us a whopping 36 bytes!

This clearly is not worth the trouble so we can remove it. The
only place where the optimization might count - in __switch_to()
- we will handle differently. But that's not subject of this
patch.

Signed-off-by: default avatarBorislav Petkov <bp@suse.de>
Cc: Andy Lutomirski <luto@amacapital.net>
Cc: Borislav Petkov <bp@alien8.de>
Cc: Brian Gerst <brgerst@gmail.com>
Cc: Denys Vlasenko <dvlasenk@redhat.com>
Cc: H. Peter Anvin <hpa@zytor.com>
Cc: Linus Torvalds <torvalds@linux-foundation.org>
Cc: Peter Zijlstra <peterz@infradead.org>
Cc: Thomas Gleixner <tglx@linutronix.de>
Link: http://lkml.kernel.org/r/1453842730-28463-6-git-send-email-bp@alien8.de


Signed-off-by: default avatarIngo Molnar <mingo@kernel.org>
parent cd4d09ec

arch/x86/Kconfig.debug

+0 −10
Original line number Diff line number Diff line
@@ -350,16 +350,6 @@ config DEBUG_IMR_SELFTEST 


	  If unsure say N here.



config X86_DEBUG_STATIC_CPU_HAS

	bool "Debug alternatives"

	depends on DEBUG_KERNEL

	---help---

	  This option causes additional code to be generated which

	  fails if static_cpu_has() is used before alternatives have

	  run.



	  If unsure, say N.



config X86_DEBUG_FPU

	bool "Debug the x86 FPU code"

	depends on DEBUG_KERNEL

arch/x86/include/asm/cpufeature.h

+7 −93
Original line number Diff line number Diff line
@@ -125,103 +125,19 @@ extern const char * const x86_bug_flags[NBUGINTS*32]; 
#define cpu_has_osxsave		boot_cpu_has(X86_FEATURE_OSXSAVE)

#define cpu_has_hypervisor	boot_cpu_has(X86_FEATURE_HYPERVISOR)

/*

 * Do not add any more of those clumsy macros - use static_cpu_has_safe() for

 * Do not add any more of those clumsy macros - use static_cpu_has() for

 * fast paths and boot_cpu_has() otherwise!

 */



#if __GNUC__ >= 4 && defined(CONFIG_X86_FAST_FEATURE_TESTS)

extern void warn_pre_alternatives(void);

extern bool __static_cpu_has_safe(u16 bit);

extern bool __static_cpu_has(u16 bit);



/*

 * Static testing of CPU features.  Used the same as boot_cpu_has().

 * These are only valid after alternatives have run, but will statically

 * patch the target code for additional performance.

 */

static __always_inline __pure bool __static_cpu_has(u16 bit)

{

#ifdef CC_HAVE_ASM_GOTO



#ifdef CONFIG_X86_DEBUG_STATIC_CPU_HAS



		/*

		 * Catch too early usage of this before alternatives

		 * have run.

		 */

		asm_volatile_goto("1: jmp %l[t_warn]\n"

			 "2:\n"

			 ".section .altinstructions,\"a\"\n"

			 " .long 1b - .\n"

			 " .long 0\n"		/* no replacement */

			 " .word %P0\n"		/* 1: do replace */

			 " .byte 2b - 1b\n"	/* source len */

			 " .byte 0\n"		/* replacement len */

			 " .byte 0\n"		/* pad len */

			 ".previous\n"

			 /* skipping size check since replacement size = 0 */

			 : : "i" (X86_FEATURE_ALWAYS) : : t_warn);



#endif



		asm_volatile_goto("1: jmp %l[t_no]\n"

			 "2:\n"

			 ".section .altinstructions,\"a\"\n"

			 " .long 1b - .\n"

			 " .long 0\n"		/* no replacement */

			 " .word %P0\n"		/* feature bit */

			 " .byte 2b - 1b\n"	/* source len */

			 " .byte 0\n"		/* replacement len */

			 " .byte 0\n"		/* pad len */

			 ".previous\n"

			 /* skipping size check since replacement size = 0 */

			 : : "i" (bit) : : t_no);

		return true;

	t_no:

		return false;



#ifdef CONFIG_X86_DEBUG_STATIC_CPU_HAS

	t_warn:

		warn_pre_alternatives();

		return false;

#endif



#else /* CC_HAVE_ASM_GOTO */



		u8 flag;

		/* Open-coded due to __stringify() in ALTERNATIVE() */

		asm volatile("1: movb $0,%0\n"

			     "2:\n"

			     ".section .altinstructions,\"a\"\n"

			     " .long 1b - .\n"

			     " .long 3f - .\n"

			     " .word %P1\n"		/* feature bit */

			     " .byte 2b - 1b\n"		/* source len */

			     " .byte 4f - 3f\n"		/* replacement len */

			     " .byte 0\n"		/* pad len */

			     ".previous\n"

			     ".section .discard,\"aw\",@progbits\n"

			     " .byte 0xff + (4f-3f) - (2b-1b)\n" /* size check */

			     ".previous\n"

			     ".section .altinstr_replacement,\"ax\"\n"

			     "3: movb $1,%0\n"

			     "4:\n"

			     ".previous\n"

			     : "=qm" (flag) : "i" (bit));

		return flag;



#endif /* CC_HAVE_ASM_GOTO */

}



#define static_cpu_has(bit)					\

(								\

	__builtin_constant_p(boot_cpu_has(bit)) ?		\

		boot_cpu_has(bit) :				\

	__builtin_constant_p(bit) ?				\

		__static_cpu_has(bit) :				\

		boot_cpu_has(bit)				\

)



static __always_inline __pure bool _static_cpu_has_safe(u16 bit)

static __always_inline __pure bool _static_cpu_has(u16 bit)

{

#ifdef CC_HAVE_ASM_GOTO

		asm_volatile_goto("1: jmp %l[t_dynamic]\n"
@@ -255,7 +171,7 @@ static __always_inline __pure bool _static_cpu_has_safe(u16 bit) 
	t_no:

		return false;

	t_dynamic:

		return __static_cpu_has_safe(bit);

		return __static_cpu_has(bit);

#else

		u8 flag;

		/* Open-coded due to __stringify() in ALTERNATIVE() */
@@ -293,22 +209,21 @@ static __always_inline __pure bool _static_cpu_has_safe(u16 bit) 
			     ".previous\n"

			     : "=qm" (flag)

			     : "i" (bit), "i" (X86_FEATURE_ALWAYS));

		return (flag == 2 ? __static_cpu_has_safe(bit) : flag);

		return (flag == 2 ? __static_cpu_has(bit) : flag);

#endif /* CC_HAVE_ASM_GOTO */

}



#define static_cpu_has_safe(bit)				\

#define static_cpu_has(bit)					\

(								\

	__builtin_constant_p(boot_cpu_has(bit)) ?		\

		boot_cpu_has(bit) :				\

		_static_cpu_has_safe(bit)			\

		_static_cpu_has(bit)				\

)

#else

/*

 * gcc 3.x is too stupid to do the static test; fall back to dynamic.

 */

#define static_cpu_has(bit)		boot_cpu_has(bit)

#define static_cpu_has_safe(bit)	boot_cpu_has(bit)

#endif



#define cpu_has_bug(c, bit)		cpu_has(c, (bit))
@@ -316,7 +231,6 @@ static __always_inline __pure bool _static_cpu_has_safe(u16 bit) 
#define clear_cpu_bug(c, bit)		clear_cpu_cap(c, (bit))



#define static_cpu_has_bug(bit)		static_cpu_has((bit))

#define static_cpu_has_bug_safe(bit)	static_cpu_has_safe((bit))

#define boot_cpu_has_bug(bit)		cpu_has_bug(&boot_cpu_data, (bit))



#define MAX_CPU_FEATURES		(NCAPINTS * 32)

arch/x86/include/asm/fpu/internal.h

+7 −7
Original line number Diff line number Diff line
@@ -59,22 +59,22 @@ extern u64 fpu__get_supported_xfeatures_mask(void); 
 */

static __always_inline __pure bool use_eager_fpu(void)

{

	return static_cpu_has_safe(X86_FEATURE_EAGER_FPU);

	return static_cpu_has(X86_FEATURE_EAGER_FPU);

}



static __always_inline __pure bool use_xsaveopt(void)

{

	return static_cpu_has_safe(X86_FEATURE_XSAVEOPT);

	return static_cpu_has(X86_FEATURE_XSAVEOPT);

}



static __always_inline __pure bool use_xsave(void)

{

	return static_cpu_has_safe(X86_FEATURE_XSAVE);

	return static_cpu_has(X86_FEATURE_XSAVE);

}



static __always_inline __pure bool use_fxsr(void)

{

	return static_cpu_has_safe(X86_FEATURE_FXSR);

	return static_cpu_has(X86_FEATURE_FXSR);

}



/*
@@ -301,7 +301,7 @@ static inline void copy_xregs_to_kernel_booting(struct xregs_state *xstate) 


	WARN_ON(system_state != SYSTEM_BOOTING);



	if (static_cpu_has_safe(X86_FEATURE_XSAVES))

	if (static_cpu_has(X86_FEATURE_XSAVES))

		XSTATE_OP(XSAVES, xstate, lmask, hmask, err);

	else

		XSTATE_OP(XSAVE, xstate, lmask, hmask, err);
@@ -323,7 +323,7 @@ static inline void copy_kernel_to_xregs_booting(struct xregs_state *xstate) 


	WARN_ON(system_state != SYSTEM_BOOTING);



	if (static_cpu_has_safe(X86_FEATURE_XSAVES))

	if (static_cpu_has(X86_FEATURE_XSAVES))

		XSTATE_OP(XRSTORS, xstate, lmask, hmask, err);

	else

		XSTATE_OP(XRSTOR, xstate, lmask, hmask, err);
@@ -461,7 +461,7 @@ static inline void copy_kernel_to_fpregs(union fpregs_state *fpstate) 
	 * pending. Clear the x87 state here by setting it to fixed values.

	 * "m" is a random variable that should be in L1.

	 */

	if (unlikely(static_cpu_has_bug_safe(X86_BUG_FXSAVE_LEAK))) {

	if (unlikely(static_cpu_has_bug(X86_BUG_FXSAVE_LEAK))) {

		asm volatile(

			"fnclex\n\t"

			"emms\n\t"

arch/x86/kernel/apic/apic_numachip.c

+2 −2
Original line number Diff line number Diff line
@@ -30,7 +30,7 @@ static unsigned int numachip1_get_apic_id(unsigned long x) 
	unsigned long value;

	unsigned int id = (x >> 24) & 0xff;



	if (static_cpu_has_safe(X86_FEATURE_NODEID_MSR)) {

	if (static_cpu_has(X86_FEATURE_NODEID_MSR)) {

		rdmsrl(MSR_FAM10H_NODE_ID, value);

		id |= (value << 2) & 0xff00;

	}
@@ -178,7 +178,7 @@ static void fixup_cpu_id(struct cpuinfo_x86 *c, int node) 
	this_cpu_write(cpu_llc_id, node);



	/* Account for nodes per socket in multi-core-module processors */

	if (static_cpu_has_safe(X86_FEATURE_NODEID_MSR)) {

	if (static_cpu_has(X86_FEATURE_NODEID_MSR)) {

		rdmsrl(MSR_FAM10H_NODE_ID, val);

		nodes = ((val >> 3) & 7) + 1;

	}

arch/x86/kernel/cpu/common.c

+2 −10
Original line number Diff line number Diff line
@@ -1475,19 +1475,11 @@ void cpu_init(void) 
}

#endif



#ifdef CONFIG_X86_DEBUG_STATIC_CPU_HAS

void warn_pre_alternatives(void)

{

	WARN(1, "You're using static_cpu_has before alternatives have run!\n");

}

EXPORT_SYMBOL_GPL(warn_pre_alternatives);

#endif



inline bool __static_cpu_has_safe(u16 bit)

inline bool __static_cpu_has(u16 bit)

{

	return boot_cpu_has(bit);

}

EXPORT_SYMBOL_GPL(__static_cpu_has_safe);

EXPORT_SYMBOL_GPL(__static_cpu_has);



static void bsp_resume(void)

{

CRA Git | Maintained and supported by SUSTech CRA and CCSE