Commit 5b06fd3b authored by Peter Zijlstra's avatar Peter Zijlstra Committed by Ingo Molnar
Browse files

static_call: Handle tail-calls 



GCC can turn our static_call(name)(args...) into a tail call, in which
case we get a JMP.d32 into the trampoline (which then does a further
tail-call).

Teach objtool to recognise and mark these in .static_call_sites and
adjust the code patching to deal with this.

Signed-off-by: default avatarPeter Zijlstra (Intel) <peterz@infradead.org>
Signed-off-by: default avatarIngo Molnar <mingo@kernel.org>
Cc: Linus Torvalds <torvalds@linux-foundation.org>
Link: https://lore.kernel.org/r/20200818135805.101186767@infradead.org
parent 452cddbf

arch/x86/kernel/static_call.c

+18 −3
Original line number Diff line number Diff line
@@ -41,15 +41,30 @@ static void __static_call_transform(void *insn, enum insn_type type, void *func) 
	text_poke_bp(insn, code, size, NULL);

}



void arch_static_call_transform(void *site, void *tramp, void *func)

static inline enum insn_type __sc_insn(bool null, bool tail)

{

	/*

	 * Encode the following table without branches:

	 *

	 *	tail	null	insn

	 *	-----+-------+------

	 *	  0  |   0   |  CALL

	 *	  0  |   1   |  NOP

	 *	  1  |   0   |  JMP

	 *	  1  |   1   |  RET

	 */

	return 2*tail + null;

}



void arch_static_call_transform(void *site, void *tramp, void *func, bool tail)

{

	mutex_lock(&text_mutex);



	if (tramp)

		__static_call_transform(tramp, func ? JMP : RET, func);

		__static_call_transform(tramp, __sc_insn(!func, true), func);



	if (IS_ENABLED(CONFIG_HAVE_STATIC_CALL_INLINE) && site)

		__static_call_transform(site, func ? CALL : NOP, func);

		__static_call_transform(site, __sc_insn(!func, tail), func);



	mutex_unlock(&text_mutex);

}

include/linux/static_call.h

+2 −2
Original line number Diff line number Diff line
@@ -103,7 +103,7 @@ 
/*

 * Either @site or @tramp can be NULL.

 */

extern void arch_static_call_transform(void *site, void *tramp, void *func);

extern void arch_static_call_transform(void *site, void *tramp, void *func, bool tail);



#define STATIC_CALL_TRAMP_ADDR(name) &STATIC_CALL_TRAMP(name)
@@ -206,7 +206,7 @@ void __static_call_update(struct static_call_key *key, void *tramp, void *func) 
{

	cpus_read_lock();

	WRITE_ONCE(key->func, func);

	arch_static_call_transform(NULL, tramp, func);

	arch_static_call_transform(NULL, tramp, func, false);

	cpus_read_unlock();

}

include/linux/static_call_types.h

+7 −0
Original line number Diff line number Diff line
@@ -16,6 +16,13 @@ 
#define STATIC_CALL_TRAMP(name)		__PASTE(STATIC_CALL_TRAMP_PREFIX, name)

#define STATIC_CALL_TRAMP_STR(name)	__stringify(STATIC_CALL_TRAMP(name))



/*

 * Flags in the low bits of static_call_site::key.

 */

#define STATIC_CALL_SITE_TAIL 1UL	/* tail call */

#define STATIC_CALL_SITE_INIT 2UL	/* init section */

#define STATIC_CALL_SITE_FLAGS 3UL



/*

 * The static call site table needs to be created by external tooling (objtool

 * or a compiler plugin).

kernel/static_call.c

+13 −8
Original line number Diff line number Diff line
@@ -15,8 +15,6 @@ extern struct static_call_site __start_static_call_sites[], 


static bool static_call_initialized;



#define STATIC_CALL_INIT 1UL



/* mutex to protect key modules/sites */

static DEFINE_MUTEX(static_call_mutex);
@@ -39,18 +37,23 @@ static inline void *static_call_addr(struct static_call_site *site) 
static inline struct static_call_key *static_call_key(const struct static_call_site *site)

{

	return (struct static_call_key *)

		(((long)site->key + (long)&site->key) & ~STATIC_CALL_INIT);

		(((long)site->key + (long)&site->key) & ~STATIC_CALL_SITE_FLAGS);

}



/* These assume the key is word-aligned. */

static inline bool static_call_is_init(struct static_call_site *site)

{

	return ((long)site->key + (long)&site->key) & STATIC_CALL_INIT;

	return ((long)site->key + (long)&site->key) & STATIC_CALL_SITE_INIT;

}



static inline bool static_call_is_tail(struct static_call_site *site)

{

	return ((long)site->key + (long)&site->key) & STATIC_CALL_SITE_TAIL;

}



static inline void static_call_set_init(struct static_call_site *site)

{

	site->key = ((long)static_call_key(site) | STATIC_CALL_INIT) -

	site->key = ((long)static_call_key(site) | STATIC_CALL_SITE_INIT) -

		    (long)&site->key;

}
@@ -104,7 +107,7 @@ void __static_call_update(struct static_call_key *key, void *tramp, void *func) 


	key->func = func;



	arch_static_call_transform(NULL, tramp, func);

	arch_static_call_transform(NULL, tramp, func, false);



	/*

	 * If uninitialized, we'll not update the callsites, but they still
@@ -154,7 +157,8 @@ void __static_call_update(struct static_call_key *key, void *tramp, void *func) 
				continue;

			}



			arch_static_call_transform(site_addr, NULL, func);

			arch_static_call_transform(site_addr, NULL, func,

				static_call_is_tail(site));

		}

	}
@@ -198,7 +202,8 @@ static int __static_call_init(struct module *mod, 
			key->mods = site_mod;

		}



		arch_static_call_transform(site_addr, NULL, key->func);

		arch_static_call_transform(site_addr, NULL, key->func,

				static_call_is_tail(site));

	}



	return 0;

tools/include/linux/static_call_types.h

+7 −0
Original line number Diff line number Diff line
@@ -16,6 +16,13 @@ 
#define STATIC_CALL_TRAMP(name)		__PASTE(STATIC_CALL_TRAMP_PREFIX, name)

#define STATIC_CALL_TRAMP_STR(name)	__stringify(STATIC_CALL_TRAMP(name))



/*

 * Flags in the low bits of static_call_site::key.

 */

#define STATIC_CALL_SITE_TAIL 1UL	/* tail call */

#define STATIC_CALL_SITE_INIT 2UL	/* init section */

#define STATIC_CALL_SITE_FLAGS 3UL



/*

 * The static call site table needs to be created by external tooling (objtool

 * or a compiler plugin).

CRA Git | Maintained and supported by SUSTech CRA and CCSE