Merge branch 'x86/jumplabel' of git://git.kernel.org/pub/scm/linux/kernel/git/tip/tip (442e0973) · Commits · 戴 / test

arch/x86/include/asm/jump_label.h

+7 −2

Original line number	Diff line number	Diff line
		@@ -3,18 +3,23 @@

		#ifdef __KERNEL__

		#include <linux/stringify.h>
		#include <linux/types.h>
		#include <asm/nops.h>
		#include <asm/asm.h>

		#define JUMP_LABEL_NOP_SIZE 5

		#define STATIC_KEY_INITIAL_NOP ".byte 0xe9 \n\t .long 0\n\t"
		#ifdef CONFIG_X86_64
		# define STATIC_KEY_INIT_NOP P6_NOP5_ATOMIC
		#else
		# define STATIC_KEY_INIT_NOP GENERIC_NOP5_ATOMIC
		#endif

		static __always_inline bool arch_static_branch(struct static_key *key)
		{
		asm goto("1:"
		STATIC_KEY_INITIAL_NOP
		".byte " __stringify(STATIC_KEY_INIT_NOP) "\n\t"
		".pushsection __jump_table, \"aw\" \n\t"
		_ASM_ALIGN "\n\t"
		_ASM_PTR "1b, %l[l_yes], %c0 \n\t"

arch/x86/kernel/jump_label.c

+66 −4

Original line number	Diff line number	Diff line
		@@ -24,18 +24,57 @@ union jump_code_union {
		} __attribute__((packed));
		};

		static void bug_at(unsigned char *ip, int line)
		{
		/*
		* The location is not an op that we were expecting.
		* Something went wrong. Crash the box, as something could be
		* corrupting the kernel.
		*/
		pr_warning("Unexpected op at %pS [%p] (%02x %02x %02x %02x %02x) %s:%d\n",
		ip, ip, ip[0], ip[1], ip[2], ip[3], ip[4], __FILE__, line);
		BUG();
		}

		static void __jump_label_transform(struct jump_entry *entry,
		enum jump_label_type type,
		void (poker)(void , const void , size_t))
		void (poker)(void , const void , size_t),
		int init)
		{
		union jump_code_union code;
		const unsigned char *ideal_nop = ideal_nops[NOP_ATOMIC5];

		if (type == JUMP_LABEL_ENABLE) {
		/*
		* We are enabling this jump label. If it is not a nop
		* then something must have gone wrong.
		*/
		if (unlikely(memcmp((void *)entry->code, ideal_nop, 5) != 0))
		bug_at((void *)entry->code, __LINE__);

		code.jump = 0xe9;
		code.offset = entry->target -
		(entry->code + JUMP_LABEL_NOP_SIZE);
		} else
		} else {
		/*
		* We are disabling this jump label. If it is not what
		* we think it is, then something must have gone wrong.
		* If this is the first initialization call, then we
		* are converting the default nop to the ideal nop.
		*/
		if (init) {
		const unsigned char default_nop[] = { STATIC_KEY_INIT_NOP };
		if (unlikely(memcmp((void *)entry->code, default_nop, 5) != 0))
		bug_at((void *)entry->code, __LINE__);
		} else {
		code.jump = 0xe9;
		code.offset = entry->target -
		(entry->code + JUMP_LABEL_NOP_SIZE);
		if (unlikely(memcmp((void *)entry->code, &code, 5) != 0))
		bug_at((void *)entry->code, __LINE__);
		}
		memcpy(&code, ideal_nops[NOP_ATOMIC5], JUMP_LABEL_NOP_SIZE);
		}

		/*
		* Make text_poke_bp() a default fallback poker.
		@@ -57,15 +96,38 @@ void arch_jump_label_transform(struct jump_entry *entry,
		{
		get_online_cpus();
		mutex_lock(&text_mutex);
		__jump_label_transform(entry, type, NULL);
		__jump_label_transform(entry, type, NULL, 0);
		mutex_unlock(&text_mutex);
		put_online_cpus();
		}

		static enum {
		JL_STATE_START,
		JL_STATE_NO_UPDATE,
		JL_STATE_UPDATE,
		} jlstate __initdata_or_module = JL_STATE_START;

		__init_or_module void arch_jump_label_transform_static(struct jump_entry *entry,
		enum jump_label_type type)
		{
		__jump_label_transform(entry, type, text_poke_early);
		/*
		* This function is called at boot up and when modules are
		* first loaded. Check if the default nop, the one that is
		* inserted at compile time, is the ideal nop. If it is, then
		* we do not need to update the nop, and we can leave it as is.
		* If it is not, then we need to update the nop to the ideal nop.
		*/
		if (jlstate == JL_STATE_START) {
		const unsigned char default_nop[] = { STATIC_KEY_INIT_NOP };
		const unsigned char *ideal_nop = ideal_nops[NOP_ATOMIC5];

		if (memcmp(ideal_nop, default_nop, 5) != 0)
		jlstate = JL_STATE_UPDATE;
		else
		jlstate = JL_STATE_NO_UPDATE;
		}
		if (jlstate == JL_STATE_UPDATE)
		__jump_label_transform(entry, type, text_poke_early, 1);
		}

		#endif

Admin message