Commit 1e7e4788 authored by Josh Poimboeuf's avatar Josh Poimboeuf Committed by Ingo Molnar
Browse files

x86/static_call: Add inline static call implementation for x86-64 



Add the inline static call implementation for x86-64. The generated code
is identical to the out-of-line case, except we move the trampoline into
it's own section.

Objtool uses the trampoline naming convention to detect all the call
sites. It then annotates those call sites in the .static_call_sites
section.

During boot (and module init), the call sites are patched to call
directly into the destination function.  The temporary trampoline is
then no longer used.

[peterz: merged trampolines, put trampoline in section]

Signed-off-by: default avatarJosh Poimboeuf <jpoimboe@redhat.com>
Signed-off-by: default avatarPeter Zijlstra (Intel) <peterz@infradead.org>
Signed-off-by: default avatarIngo Molnar <mingo@kernel.org>
Cc: Linus Torvalds <torvalds@linux-foundation.org>
Link: https://lore.kernel.org/r/20200818135804.864271425@infradead.org
parent e6d6c071

arch/x86/Kconfig

+2 −1
Original line number Diff line number Diff line
@@ -216,6 +216,7 @@ config X86 
	select HAVE_STACKPROTECTOR		if CC_HAS_SANE_STACKPROTECTOR

	select HAVE_STACK_VALIDATION		if X86_64

	select HAVE_STATIC_CALL

	select HAVE_STATIC_CALL_INLINE		if HAVE_STACK_VALIDATION

	select HAVE_RSEQ

	select HAVE_SYSCALL_TRACEPOINTS

	select HAVE_UNSTABLE_SCHED_CLOCK
@@ -231,6 +232,7 @@ config X86 
	select RTC_MC146818_LIB

	select SPARSE_IRQ

	select SRCU

	select STACK_VALIDATION			if HAVE_STACK_VALIDATION && (HAVE_STATIC_CALL_INLINE || RETPOLINE)

	select SYSCTL_EXCEPTION_TRACE

	select THREAD_INFO_IN_TASK

	select USER_STACKTRACE_SUPPORT
@@ -452,7 +454,6 @@ config GOLDFISH 
config RETPOLINE

	bool "Avoid speculative indirect branches in kernel"

	default y

	select STACK_VALIDATION if HAVE_STACK_VALIDATION

	help

	  Compile kernel with the retpoline compiler options to guard against

	  kernel-to-user data leaks by avoiding speculative indirect

arch/x86/include/asm/static_call.h

+12 −1
Original line number Diff line number Diff line
@@ -5,12 +5,23 @@ 
#include <asm/text-patching.h>



/*

 * For CONFIG_HAVE_STATIC_CALL_INLINE, this is a temporary trampoline which

 * uses the current value of the key->func pointer to do an indirect jump to

 * the function.  This trampoline is only used during boot, before the call

 * sites get patched by static_call_update().  The name of this trampoline has

 * a magical aspect: objtool uses it to find static call sites so it can create

 * the .static_call_sites section.

 *

 * For CONFIG_HAVE_STATIC_CALL, this is a permanent trampoline which

 * does a direct jump to the function.  The direct jump gets patched by

 * static_call_update().

 *

 * Having the trampoline in a special section forces GCC to emit a JMP.d32 when

 * it does tail-call optimization on the call; since you cannot compute the

 * relative displacement across sections.

 */

#define ARCH_DEFINE_STATIC_CALL_TRAMP(name, func)			\

	asm(".pushsection .text, \"ax\"				\n"	\

	asm(".pushsection .static_call.text, \"ax\"		\n"	\

	    ".align 4						\n"	\

	    ".globl " STATIC_CALL_TRAMP_STR(name) "		\n"	\

	    STATIC_CALL_TRAMP_STR(name) ":			\n"	\

arch/x86/kernel/static_call.c

+3 −0
Original line number Diff line number Diff line
@@ -26,6 +26,9 @@ void arch_static_call_transform(void *site, void *tramp, void *func) 
	if (tramp)

		__static_call_transform(tramp, JMP32_INSN_OPCODE, func);



	if (IS_ENABLED(CONFIG_HAVE_STATIC_CALL_INLINE) && site)

		__static_call_transform(site, CALL_INSN_OPCODE, func);



	mutex_unlock(&text_mutex);

}

EXPORT_SYMBOL_GPL(arch_static_call_transform);

arch/x86/kernel/vmlinux.lds.S

+1 −0
Original line number Diff line number Diff line
@@ -136,6 +136,7 @@ SECTIONS 
		ENTRY_TEXT

		ALIGN_ENTRY_TEXT_END

		SOFTIRQENTRY_TEXT

		STATIC_CALL_TEXT

		*(.fixup)

		*(.gnu.warning)

include/asm-generic/vmlinux.lds.h

+6 −0
Original line number Diff line number Diff line
@@ -642,6 +642,12 @@ 
		*(.softirqentry.text)					\

		__softirqentry_text_end = .;



#define STATIC_CALL_TEXT						\

		ALIGN_FUNCTION();					\

		__static_call_text_start = .;				\

		*(.static_call.text)					\

		__static_call_text_end = .;



/* Section used for early init (in .S files) */

#define HEAD_TEXT  KEEP(*(.head.text))

CRA Git | Maintained and supported by SUSTech CRA and CCSE