Commit f81f8ad5 authored by Nadav Amit's avatar Nadav Amit Committed by Ingo Molnar
Browse files

x86/bug: Macrofy the BUG table section handling, to work around GCC inlining bugs 



As described in:

  77b0bf55: ("kbuild/Makefile: Prepare for using macros in inline assembly code to work around asm() related GCC inlining bugs")

GCC's inlining heuristics are broken with common asm() patterns used in
kernel code, resulting in the effective disabling of inlining.

The workaround is to set an assembly macro and call it from the inline
assembly block. As a result GCC considers the inline assembly block as
a single instruction. (Which it isn't, but that's the best we can get.)

This patch increases the kernel size:

      text     data     bss      dec     hex  filename
  18146889 10225380 2957312 31329581 1de0d2d  ./vmlinux before
  18147336 10226688 2957312 31331336 1de1408  ./vmlinux after (+1755)

But enables more aggressive inlining (and probably better branch decisions).

The number of static text symbols in vmlinux is much lower:

 Before: 40218
 After:  40053 (-165)

The assembly code gets harder to read due to the extra macro layer.

[ mingo: Rewrote the changelog. ]

Tested-by: default avatarKees Cook <keescook@chromium.org>
Signed-off-by: default avatarNadav Amit <namit@vmware.com>
Acked-by: default avatarPeter Zijlstra (Intel) <peterz@infradead.org>
Cc: Andy Lutomirski <luto@amacapital.net>
Cc: Borislav Petkov <bp@alien8.de>
Cc: Brian Gerst <brgerst@gmail.com>
Cc: Denys Vlasenko <dvlasenk@redhat.com>
Cc: H. Peter Anvin <hpa@zytor.com>
Cc: Josh Poimboeuf <jpoimboe@redhat.com>
Cc: Linus Torvalds <torvalds@linux-foundation.org>
Cc: Peter Zijlstra <peterz@infradead.org>
Cc: Thomas Gleixner <tglx@linutronix.de>
Link: http://lkml.kernel.org/r/20181003213100.189959-7-namit@vmware.com


Signed-off-by: default avatarIngo Molnar <mingo@kernel.org>
parent 77f48ec2

arch/x86/include/asm/bug.h

+56 −42
Original line number Original line Diff line number Diff line
@@ -4,6 +4,8 @@ 




#include <linux/stringify.h>

#include <linux/stringify.h>





#ifndef __ASSEMBLY__



/*

/*

 * Despite that some emulators terminate on UD2, we use it for WARN().

 * Despite that some emulators terminate on UD2, we use it for WARN().

 *

 *
@@ -20,53 +22,15 @@ 




#define LEN_UD2		2

#define LEN_UD2		2





#ifdef CONFIG_GENERIC_BUG



#ifdef CONFIG_X86_32

# define __BUG_REL(val)	".long " __stringify(val)

#else

# define __BUG_REL(val)	".long " __stringify(val) " - 2b"

#endif



#ifdef CONFIG_DEBUG_BUGVERBOSE



#define _BUG_FLAGS(ins, flags)						\

#define _BUG_FLAGS(ins, flags)						\

do {									\

do {									\

	asm volatile("1:\t" ins "\n"					\

	asm volatile("ASM_BUG ins=\"" ins "\" file=%c0 line=%c1 "	\

		     ".pushsection __bug_table,\"aw\"\n"		\

		     "flags=%c2 size=%c3"				\

		     "2:\t" __BUG_REL(1b) "\t# bug_entry::bug_addr\n"	\

		     "\t"  __BUG_REL(%c0) "\t# bug_entry::file\n"	\

		     "\t.word %c1"        "\t# bug_entry::line\n"	\

		     "\t.word %c2"        "\t# bug_entry::flags\n"	\

		     "\t.org 2b+%c3\n"					\

		     ".popsection"					\

		     : : "i" (__FILE__), "i" (__LINE__),                \

		     : : "i" (__FILE__), "i" (__LINE__),                \

			 "i" (flags),                                   \

			 "i" (flags),                                   \

			 "i" (sizeof(struct bug_entry)));		\

			 "i" (sizeof(struct bug_entry)));		\

} while (0)

} while (0)





#else /* !CONFIG_DEBUG_BUGVERBOSE */



#define _BUG_FLAGS(ins, flags)						\

do {									\

	asm volatile("1:\t" ins "\n"					\

		     ".pushsection __bug_table,\"aw\"\n"		\

		     "2:\t" __BUG_REL(1b) "\t# bug_entry::bug_addr\n"	\

		     "\t.word %c0"        "\t# bug_entry::flags\n"	\

		     "\t.org 2b+%c1\n"					\

		     ".popsection"					\

		     : : "i" (flags),					\

			 "i" (sizeof(struct bug_entry)));		\

} while (0)



#endif /* CONFIG_DEBUG_BUGVERBOSE */



#else



#define _BUG_FLAGS(ins, flags)  asm volatile(ins)



#endif /* CONFIG_GENERIC_BUG */



#define HAVE_ARCH_BUG

#define HAVE_ARCH_BUG

#define BUG()							\

#define BUG()							\

do {								\

do {								\
@@ -82,4 +46,54 @@ do { \ 




#include <asm-generic/bug.h>

#include <asm-generic/bug.h>





#else /* __ASSEMBLY__ */



#ifdef CONFIG_GENERIC_BUG



#ifdef CONFIG_X86_32

.macro __BUG_REL val:req

	.long \val

.endm

#else

.macro __BUG_REL val:req

	.long \val - 2b

.endm

#endif



#ifdef CONFIG_DEBUG_BUGVERBOSE



.macro ASM_BUG ins:req file:req line:req flags:req size:req

1:	\ins

	.pushsection __bug_table,"aw"

2:	__BUG_REL val=1b	# bug_entry::bug_addr

	__BUG_REL val=\file	# bug_entry::file

	.word \line		# bug_entry::line

	.word \flags		# bug_entry::flags

	.org 2b+\size

	.popsection

.endm



#else /* !CONFIG_DEBUG_BUGVERBOSE */



.macro ASM_BUG ins:req file:req line:req flags:req size:req

1:	\ins

	.pushsection __bug_table,"aw"

2:	__BUG_REL val=1b	# bug_entry::bug_addr

	.word \flags		# bug_entry::flags

	.org 2b+\size

	.popsection

.endm



#endif /* CONFIG_DEBUG_BUGVERBOSE */



#else /* CONFIG_GENERIC_BUG */



.macro ASM_BUG ins:req file:req line:req flags:req size:req

	\ins

.endm



#endif /* CONFIG_GENERIC_BUG */



#endif /* __ASSEMBLY__ */



#endif /* _ASM_X86_BUG_H */
 
#endif /* _ASM_X86_BUG_H */

arch/x86/kernel/macros.S

+1 −0
Original line number Original line Diff line number Diff line
@@ -9,3 +9,4 @@ 
#include <linux/compiler.h>

#include <linux/compiler.h>

#include <asm/refcount.h>

#include <asm/refcount.h>

#include <asm/alternative-asm.h>

#include <asm/alternative-asm.h>

#include <asm/bug.h>

include/asm-generic/bug.h

+4 −4
Original line number Original line Diff line number Diff line
@@ -17,10 +17,8 @@ 
#ifndef __ASSEMBLY__

#ifndef __ASSEMBLY__

#include <linux/kernel.h>

#include <linux/kernel.h>





#ifdef CONFIG_BUG



#ifdef CONFIG_GENERIC_BUG

struct bug_entry {

struct bug_entry {

#ifdef CONFIG_GENERIC_BUG

#ifndef CONFIG_GENERIC_BUG_RELATIVE_POINTERS

#ifndef CONFIG_GENERIC_BUG_RELATIVE_POINTERS

	unsigned long	bug_addr;

	unsigned long	bug_addr;

#else

#else
@@ -35,8 +33,10 @@ struct bug_entry { 
	unsigned short	line;

	unsigned short	line;

#endif

#endif

	unsigned short	flags;

	unsigned short	flags;

};

#endif	/* CONFIG_GENERIC_BUG */

#endif	/* CONFIG_GENERIC_BUG */

};



#ifdef CONFIG_BUG





/*

/*

 * Don't use BUG() or BUG_ON() unless there's really no way out; one

 * Don't use BUG() or BUG_ON() unless there's really no way out; one

CRA Git | Maintained and supported by SUSTech CRA and CCSE