Commit 5bdcd510 authored by Nadav Amit's avatar Nadav Amit Committed by Ingo Molnar
Browse files

x86/jump-labels: Macrofy inline assembly code to work around GCC inlining bugs 



As described in:

  77b0bf55: ("kbuild/Makefile: Prepare for using macros in inline assembly code to work around asm() related GCC inlining bugs")

GCC's inlining heuristics are broken with common asm() patterns used in
kernel code, resulting in the effective disabling of inlining.

The workaround is to set an assembly macro and call it from the inline
assembly block - which is also a minor cleanup for the jump-label code.

As a result the code size is slightly increased, but inlining decisions
are better:

      text     data     bss      dec     hex  filename
  18163528 10226300 2957312 31347140 1de51c4  ./vmlinux before
  18163608 10227348 2957312 31348268 1de562c  ./vmlinux after (+1128)

And functions such as intel_pstate_adjust_policy_max(),
kvm_cpu_accept_dm_intr(), kvm_register_readl() are inlined.

Tested-by: default avatarKees Cook <keescook@chromium.org>
Signed-off-by: default avatarNadav Amit <namit@vmware.com>
Acked-by: default avatarPeter Zijlstra (Intel) <peterz@infradead.org>
Cc: Andy Lutomirski <luto@amacapital.net>
Cc: Borislav Petkov <bp@alien8.de>
Cc: Brian Gerst <brgerst@gmail.com>
Cc: Denys Vlasenko <dvlasenk@redhat.com>
Cc: Greg Kroah-Hartman <gregkh@linuxfoundation.org>
Cc: Kate Stewart <kstewart@linuxfoundation.org>
Cc: Linus Torvalds <torvalds@linux-foundation.org>
Cc: Peter Zijlstra <peterz@infradead.org>
Cc: Philippe Ombredanne <pombredanne@nexb.com>
Cc: Thomas Gleixner <tglx@linutronix.de>
Link: http://lkml.kernel.org/r/20181005202718.229565-4-namit@vmware.com
Link: https://lore.kernel.org/lkml/20181003213100.189959-11-namit@vmware.com/T/#u


Signed-off-by: default avatarIngo Molnar <mingo@kernel.org>
parent d5a581d8

arch/x86/entry/calling.h

+1 −1
Original line number Diff line number Diff line
@@ -338,7 +338,7 @@ For 32-bit we have the following conventions - kernel is built with 
.macro CALL_enter_from_user_mode

#ifdef CONFIG_CONTEXT_TRACKING

#ifdef HAVE_JUMP_LABEL

	STATIC_JUMP_IF_FALSE .Lafter_call_\@, context_tracking_enabled, def=0

	STATIC_BRANCH_JMP l_yes=.Lafter_call_\@, key=context_tracking_enabled, branch=1

#endif

	call enter_from_user_mode

.Lafter_call_\@:

arch/x86/include/asm/jump_label.h

+18 −54
Original line number Diff line number Diff line
@@ -2,19 +2,6 @@ 
#ifndef _ASM_X86_JUMP_LABEL_H

#define _ASM_X86_JUMP_LABEL_H



#ifndef HAVE_JUMP_LABEL

/*

 * For better or for worse, if jump labels (the gcc extension) are missing,

 * then the entire static branch patching infrastructure is compiled out.

 * If that happens, the code in here will malfunction.  Raise a compiler

 * error instead.

 *

 * In theory, jump labels and the static branch patching infrastructure

 * could be decoupled to fix this.

 */

#error asm/jump_label.h included on a non-jump-label kernel

#endif



#define JUMP_LABEL_NOP_SIZE 5



#ifdef CONFIG_X86_64
@@ -33,15 +20,9 @@ 


static __always_inline bool arch_static_branch(struct static_key *key, bool branch)

{

	asm_volatile_goto("1:"

		".byte " __stringify(STATIC_KEY_INIT_NOP) "\n\t"

		".pushsection __jump_table,  \"aw\" \n\t"

		_ASM_ALIGN "\n\t"

		".long 1b - ., %l[l_yes] - . \n\t"

		_ASM_PTR "%c0 + %c1 - .\n\t"

		".popsection \n\t"

	asm_volatile_goto("STATIC_BRANCH_NOP l_yes=\"%l[l_yes]\" key=\"%c0\" "

			  "branch=\"%c1\""

			: :  "i" (key), "i" (branch) : : l_yes);



	return false;

l_yes:

	return true;
@@ -49,14 +30,8 @@ l_yes: 


static __always_inline bool arch_static_branch_jump(struct static_key *key, bool branch)

{

	asm_volatile_goto("1:"

		".byte 0xe9\n\t .long %l[l_yes] - 2f\n\t"

		"2:\n\t"

		".pushsection __jump_table,  \"aw\" \n\t"

		_ASM_ALIGN "\n\t"

		".long 1b - ., %l[l_yes] - . \n\t"

		_ASM_PTR "%c0 + %c1 - .\n\t"

		".popsection \n\t"

	asm_volatile_goto("STATIC_BRANCH_JMP l_yes=\"%l[l_yes]\" key=\"%c0\" "

			  "branch=\"%c1\""

		: :  "i" (key), "i" (branch) : : l_yes);



	return false;
@@ -66,37 +41,26 @@ l_yes: 


#else	/* __ASSEMBLY__ */



.macro STATIC_JUMP_IF_TRUE target, key, def

.Lstatic_jump_\@:

	.if \def

	/* Equivalent to "jmp.d32 \target" */

	.byte		0xe9

	.long		\target - .Lstatic_jump_after_\@

.Lstatic_jump_after_\@:

	.else

.macro STATIC_BRANCH_NOP l_yes:req key:req branch:req

.Lstatic_branch_nop_\@:

	.byte STATIC_KEY_INIT_NOP

	.endif

.Lstatic_branch_no_after_\@:

	.pushsection __jump_table, "aw"

	_ASM_ALIGN

	.long		.Lstatic_jump_\@ - ., \target - .

	_ASM_PTR	\key - .

	.long		.Lstatic_branch_nop_\@ - ., \l_yes - .

	_ASM_PTR        \key + \branch - .

	.popsection

.endm



.macro STATIC_JUMP_IF_FALSE target, key, def

.Lstatic_jump_\@:

	.if \def

	.byte		STATIC_KEY_INIT_NOP

	.else

	/* Equivalent to "jmp.d32 \target" */

.macro STATIC_BRANCH_JMP l_yes:req key:req branch:req

.Lstatic_branch_jmp_\@:

	.byte 0xe9

	.long		\target - .Lstatic_jump_after_\@

.Lstatic_jump_after_\@:

	.endif

	.long \l_yes - .Lstatic_branch_jmp_after_\@

.Lstatic_branch_jmp_after_\@:

	.pushsection __jump_table, "aw"

	_ASM_ALIGN

	.long		.Lstatic_jump_\@ - ., \target - .

	_ASM_PTR	\key + 1 - .

	.long		.Lstatic_branch_jmp_\@ - ., \l_yes - .

	_ASM_PTR	\key + \branch - .

	.popsection

.endm

arch/x86/kernel/macros.S

+1 −0
Original line number Diff line number Diff line
@@ -13,3 +13,4 @@ 
#include <asm/paravirt.h>

#include <asm/asm.h>

#include <asm/cpufeature.h>

#include <asm/jump_label.h>

CRA Git | Maintained and supported by SUSTech CRA and CCSE