Commit a945c834 authored by Peter Zijlstra's avatar Peter Zijlstra Committed by Ingo Molnar
Browse files

static_call: Allow early init 



In order to use static_call() to wire up x86_pmu, we need to
initialize earlier, specifically before memory allocation works; copy
some of the tricks from jump_label to enable this.

Primarily we overload key->next to store a sites pointer when there
are no modules, this avoids having to use kmalloc() to initialize the
sites and allows us to run much earlier.

Signed-off-by: default avatarPeter Zijlstra (Intel) <peterz@infradead.org>
Signed-off-by: default avatarIngo Molnar <mingo@kernel.org>
Reviewed-by: default avatarSteven Rostedt (VMware) <rostedt@goodmis.org>
Link: https://lore.kernel.org/r/20200818135805.220737930@infradead.org
parent 6c3fce79

arch/x86/kernel/setup.c

+2 −0
Original line number Diff line number Diff line
@@ -19,6 +19,7 @@ 
#include <linux/hugetlb.h>

#include <linux/tboot.h>

#include <linux/usb/xhci-dbgp.h>

#include <linux/static_call.h>



#include <uapi/linux/mount.h>
@@ -849,6 +850,7 @@ void __init setup_arch(char **cmdline_p) 
	early_cpu_init();

	arch_init_ideal_nops();

	jump_label_init();

	static_call_init();

	early_ioremap_init();



	setup_olpc_ofw_pgd();

arch/x86/kernel/static_call.c

+4 −1
Original line number Diff line number Diff line
@@ -11,7 +11,7 @@ enum insn_type { 
	RET = 3,  /* tramp / site cond-tail-call */

};



static void __static_call_transform(void *insn, enum insn_type type, void *func)

static void __ref __static_call_transform(void *insn, enum insn_type type, void *func)

{

	int size = CALL_INSN_SIZE;

	const void *code;
@@ -38,6 +38,9 @@ static void __static_call_transform(void *insn, enum insn_type type, void *func) 
	if (memcmp(insn, code, size) == 0)

		return;



	if (unlikely(system_state == SYSTEM_BOOTING))

		return text_poke_early(insn, code, size);



	text_poke_bp(insn, code, size, NULL);

}

include/linux/static_call.h

+13 −2
Original line number Diff line number Diff line
@@ -136,6 +136,8 @@ extern void arch_static_call_transform(void *site, void *tramp, void *func, bool 


#ifdef CONFIG_HAVE_STATIC_CALL_INLINE



extern void __init static_call_init(void);



struct static_call_mod {

	struct static_call_mod *next;

	struct module *mod; /* for vmlinux, mod == NULL */
@@ -144,7 +146,12 @@ struct static_call_mod { 


struct static_call_key {

	void *func;

	union {

		/* bit 0: 0 = mods, 1 = sites */

		unsigned long type;

		struct static_call_mod *mods;

		struct static_call_site *sites;

	};

};



extern void __static_call_update(struct static_call_key *key, void *tramp, void *func);
@@ -155,7 +162,7 @@ extern int static_call_text_reserved(void *start, void *end); 
	DECLARE_STATIC_CALL(name, _func);				\

	struct static_call_key STATIC_CALL_KEY(name) = {		\

		.func = _func,						\

		.mods = NULL,						\

		.type = 1,						\

	};								\

	ARCH_DEFINE_STATIC_CALL_TRAMP(name, _func)
@@ -180,6 +187,8 @@ extern int static_call_text_reserved(void *start, void *end); 


#elif defined(CONFIG_HAVE_STATIC_CALL)



static inline void static_call_init(void) { }



struct static_call_key {

	void *func;

};
@@ -225,6 +234,8 @@ static inline int static_call_text_reserved(void *start, void *end) 


#else /* Generic implementation */



static inline void static_call_init(void) { }



struct static_call_key {

	void *func;

};

kernel/static_call.c

+66 −4
Original line number Diff line number Diff line
@@ -94,10 +94,31 @@ static inline void static_call_sort_entries(struct static_call_site *start, 
	     static_call_site_cmp, static_call_site_swap);

}



static inline bool static_call_key_has_mods(struct static_call_key *key)

{

	return !(key->type & 1);

}



static inline struct static_call_mod *static_call_key_next(struct static_call_key *key)

{

	if (!static_call_key_has_mods(key))

		return NULL;



	return key->mods;

}



static inline struct static_call_site *static_call_key_sites(struct static_call_key *key)

{

	if (static_call_key_has_mods(key))

		return NULL;



	return (struct static_call_site *)(key->type & ~1);

}



void __static_call_update(struct static_call_key *key, void *tramp, void *func)

{

	struct static_call_site *site, *stop;

	struct static_call_mod *site_mod;

	struct static_call_mod *site_mod, first;



	cpus_read_lock();

	static_call_lock();
@@ -116,13 +137,22 @@ void __static_call_update(struct static_call_key *key, void *tramp, void *func) 
	if (WARN_ON_ONCE(!static_call_initialized))

		goto done;



	for (site_mod = key->mods; site_mod; site_mod = site_mod->next) {

	first = (struct static_call_mod){

		.next = static_call_key_next(key),

		.mod = NULL,

		.sites = static_call_key_sites(key),

	};



	for (site_mod = &first; site_mod; site_mod = site_mod->next) {

		struct module *mod = site_mod->mod;



		if (!site_mod->sites) {

			/*

			 * This can happen if the static call key is defined in

			 * a module which doesn't use it.

			 *

			 * It also happens in the has_mods case, where the

			 * 'first' entry has no sites associated with it.

			 */

			continue;

		}
@@ -192,16 +222,48 @@ static int __static_call_init(struct module *mod, 
		if (key != prev_key) {

			prev_key = key;



			/*

			 * For vmlinux (!mod) avoid the allocation by storing

			 * the sites pointer in the key itself. Also see

			 * __static_call_update()'s @first.

			 *

			 * This allows architectures (eg. x86) to call

			 * static_call_init() before memory allocation works.

			 */

			if (!mod) {

				key->sites = site;

				key->type |= 1;

				goto do_transform;

			}



			site_mod = kzalloc(sizeof(*site_mod), GFP_KERNEL);

			if (!site_mod)

				return -ENOMEM;



			/*

			 * When the key has a direct sites pointer, extract

			 * that into an explicit struct static_call_mod, so we

			 * can have a list of modules.

			 */

			if (static_call_key_sites(key)) {

				site_mod->mod = NULL;

				site_mod->next = NULL;

				site_mod->sites = static_call_key_sites(key);



				key->mods = site_mod;



				site_mod = kzalloc(sizeof(*site_mod), GFP_KERNEL);

				if (!site_mod)

					return -ENOMEM;

			}



			site_mod->mod = mod;

			site_mod->sites = site;

			site_mod->next = key->mods;

			site_mod->next = static_call_key_next(key);

			key->mods = site_mod;

		}



do_transform:

		arch_static_call_transform(site_addr, NULL, key->func,

				static_call_is_tail(site));

	}
@@ -348,7 +410,7 @@ int static_call_text_reserved(void *start, void *end) 
	return __static_call_mod_text_reserved(start, end);

}



static void __init static_call_init(void)

void __init static_call_init(void)

{

	int ret;

CRA Git | Maintained and supported by SUSTech CRA and CCSE