Commit 9183c3f9 authored by Josh Poimboeuf's avatar Josh Poimboeuf Committed by Ingo Molnar
Browse files

static_call: Add inline static call infrastructure 

Add infrastructure for an arch-specific CONFIG_HAVE_STATIC_CALL_INLINE
option, which is a faster version of CONFIG_HAVE_STATIC_CALL.  At
runtime, the static call sites are patched directly, rather than using
the out-of-line trampolines.

Compared to out-of-line static calls, the performance benefits are more
modest, but still measurable.  Steven Rostedt did some tracepoint
measurements:

  https://lkml.kernel.org/r/20181126155405.72b4f718@gandalf.local.home



This code is heavily inspired by the jump label code (aka "static
jumps"), as some of the concepts are very similar.

For more details, see the comments in include/linux/static_call.h.

[peterz: simplified interface; merged trampolines]

Signed-off-by: default avatarJosh Poimboeuf <jpoimboe@redhat.com>
Signed-off-by: default avatarPeter Zijlstra (Intel) <peterz@infradead.org>
Signed-off-by: default avatarIngo Molnar <mingo@kernel.org>
Reviewed-by: default avatarSteven Rostedt (VMware) <rostedt@goodmis.org>
Cc: Linus Torvalds <torvalds@linux-foundation.org>
Link: https://lore.kernel.org/r/20200818135804.684334440@infradead.org
parent 115284d8

arch/Kconfig

+4 −0
Original line number Diff line number Diff line
@@ -978,6 +978,10 @@ config ARCH_HAS_VDSO_DATA 
config HAVE_STATIC_CALL

	bool



config HAVE_STATIC_CALL_INLINE

	bool

	depends on HAVE_STATIC_CALL



source "kernel/gcov/Kconfig"



source "scripts/gcc-plugins/Kconfig"

include/asm-generic/vmlinux.lds.h

+7 −0
Original line number Diff line number Diff line
@@ -388,6 +388,12 @@ 
	KEEP(*(__jump_table))						\

	__stop___jump_table = .;



#define STATIC_CALL_DATA						\

	. = ALIGN(8);							\

	__start_static_call_sites = .;					\

	KEEP(*(.static_call_sites))					\

	__stop_static_call_sites = .;



/*

 * Allow architectures to handle ro_after_init data on their

 * own by defining an empty RO_AFTER_INIT_DATA.
@@ -398,6 +404,7 @@ 
	__start_ro_after_init = .;					\

	*(.data..ro_after_init)						\

	JUMP_TABLE_DATA							\

	STATIC_CALL_DATA						\

	__end_ro_after_init = .;

#endif

include/linux/module.h

+5 −0
Original line number Diff line number Diff line
@@ -25,6 +25,7 @@ 
#include <linux/error-injection.h>

#include <linux/tracepoint-defs.h>

#include <linux/srcu.h>

#include <linux/static_call_types.h>



#include <linux/percpu.h>

#include <asm/module.h>
@@ -498,6 +499,10 @@ struct module { 
	unsigned long *kprobe_blacklist;

	unsigned int num_kprobe_blacklist;

#endif

#ifdef CONFIG_HAVE_STATIC_CALL_INLINE

	int num_static_call_sites;

	struct static_call_site *static_call_sites;

#endif



#ifdef CONFIG_LIVEPATCH

	bool klp; /* Is this a livepatch module? */

include/linux/static_call.h

+35 −1
Original line number Diff line number Diff line
@@ -95,7 +95,41 @@ extern void arch_static_call_transform(void *site, void *tramp, void *func); 
			     STATIC_CALL_TRAMP_ADDR(name), func);	\

})



#if defined(CONFIG_HAVE_STATIC_CALL)

#ifdef CONFIG_HAVE_STATIC_CALL_INLINE



struct static_call_mod {

	struct static_call_mod *next;

	struct module *mod; /* for vmlinux, mod == NULL */

	struct static_call_site *sites;

};



struct static_call_key {

	void *func;

	struct static_call_mod *mods;

};



extern void __static_call_update(struct static_call_key *key, void *tramp, void *func);

extern int static_call_mod_init(struct module *mod);



#define DEFINE_STATIC_CALL(name, _func)					\

	DECLARE_STATIC_CALL(name, _func);				\

	struct static_call_key STATIC_CALL_KEY(name) = {		\

		.func = _func,						\

		.mods = NULL,						\

	};								\

	ARCH_DEFINE_STATIC_CALL_TRAMP(name, _func)



#define static_call(name)	__static_call(name)



#define EXPORT_STATIC_CALL(name)					\

	EXPORT_SYMBOL(STATIC_CALL_KEY(name));				\

	EXPORT_SYMBOL(STATIC_CALL_TRAMP(name))



#define EXPORT_STATIC_CALL_GPL(name)					\

	EXPORT_SYMBOL_GPL(STATIC_CALL_KEY(name));			\

	EXPORT_SYMBOL_GPL(STATIC_CALL_TRAMP(name))



#elif defined(CONFIG_HAVE_STATIC_CALL)



struct static_call_key {

	void *func;

include/linux/static_call_types.h

+13 −0
Original line number Diff line number Diff line
@@ -2,14 +2,27 @@ 
#ifndef _STATIC_CALL_TYPES_H

#define _STATIC_CALL_TYPES_H



#include <linux/types.h>

#include <linux/stringify.h>



#define STATIC_CALL_KEY_PREFIX		__SCK__

#define STATIC_CALL_KEY_PREFIX_STR	__stringify(STATIC_CALL_KEY_PREFIX)

#define STATIC_CALL_KEY_PREFIX_LEN	(sizeof(STATIC_CALL_KEY_PREFIX_STR) - 1)

#define STATIC_CALL_KEY(name)		__PASTE(STATIC_CALL_KEY_PREFIX, name)



#define STATIC_CALL_TRAMP_PREFIX	__SCT__

#define STATIC_CALL_TRAMP_PREFIX_STR	__stringify(STATIC_CALL_TRAMP_PREFIX)

#define STATIC_CALL_TRAMP_PREFIX_LEN	(sizeof(STATIC_CALL_TRAMP_PREFIX_STR) - 1)

#define STATIC_CALL_TRAMP(name)		__PASTE(STATIC_CALL_TRAMP_PREFIX, name)

#define STATIC_CALL_TRAMP_STR(name)	__stringify(STATIC_CALL_TRAMP(name))



/*

 * The static call site table needs to be created by external tooling (objtool

 * or a compiler plugin).

 */

struct static_call_site {

	s32 addr;

	s32 key;

};



#endif /* _STATIC_CALL_TYPES_H */

CRA Git | Maintained and supported by SUSTech CRA and CCSE