Commit b32fc0a0 authored by Linus Torvalds's avatar Linus Torvalds
Browse files

Merge branch 'upstream/jump-label-noearly' of... 

Merge branch 'upstream/jump-label-noearly' of git://git.kernel.org/pub/scm/linux/kernel/git/jeremy/xen

* 'upstream/jump-label-noearly' of git://git.kernel.org/pub/scm/linux/kernel/git/jeremy/xen:
  jump-label: initialize jump-label subsystem much earlier
  x86/jump_label: add arch_jump_label_transform_static()
  s390/jump-label: add arch_jump_label_transform_static()
  jump_label: add arch_jump_label_transform_static() to optimise non-live code updates
  sparc/jump_label: drop arch_jump_label_text_poke_early()
  x86/jump_label: drop arch_jump_label_text_poke_early()
  jump_label: if a key has already been initialized, don't nop it out
  stop_machine: make stop_machine safe and efficient to call early
  jump_label: use proper atomic_t initializer

Conflicts:
 - arch/x86/kernel/jump_label.c
	Added __init_or_module to arch_jump_label_text_poke_early vs
	removal of that function entirely
 - kernel/stop_machine.c
	same patch ("stop_machine: make stop_machine safe and efficient
	to call early") merged twice, with whitespace fix in one version
parents 403299a8 97ce2c88

arch/s390/kernel/jump_label.c

+31 −20
Original line number Diff line number Diff line
@@ -18,26 +18,15 @@ struct insn { 
} __packed;



struct insn_args {

	unsigned long *target;

	struct insn *insn;

	ssize_t size;

	struct jump_entry *entry;

	enum jump_label_type type;

};



static int __arch_jump_label_transform(void *data)

{

	struct insn_args *args = data;

	int rc;



	rc = probe_kernel_write(args->target, args->insn, args->size);

	WARN_ON_ONCE(rc < 0);

	return 0;

}



void arch_jump_label_transform(struct jump_entry *entry,

static void __jump_label_transform(struct jump_entry *entry,

				   enum jump_label_type type)

{

	struct insn_args args;

	struct insn insn;

	int rc;



	if (type == JUMP_LABEL_ENABLE) {

		/* brcl 15,offset */
@@ -49,11 +38,33 @@ void arch_jump_label_transform(struct jump_entry *entry, 
		insn.offset = 0;

	}



	args.target = (void *) entry->code;

	args.insn = &insn;

	args.size = JUMP_LABEL_NOP_SIZE;

	rc = probe_kernel_write((void *)entry->code, &insn, JUMP_LABEL_NOP_SIZE);

	WARN_ON_ONCE(rc < 0);

}



	stop_machine(__arch_jump_label_transform, &args, NULL);

static int __sm_arch_jump_label_transform(void *data)

{

	struct insn_args *args = data;



	__jump_label_transform(args->entry, args->type);

	return 0;

}



void arch_jump_label_transform(struct jump_entry *entry,

			       enum jump_label_type type)

{

	struct insn_args args;



	args.entry = entry;

	args.type = type;



	stop_machine(__sm_arch_jump_label_transform, &args, NULL);

}



void arch_jump_label_transform_static(struct jump_entry *entry,

				      enum jump_label_type type)

{

	__jump_label_transform(entry, type);

}



#endif

arch/sparc/kernel/jump_label.c

+0 −8
Original line number Diff line number Diff line
@@ -36,12 +36,4 @@ void arch_jump_label_transform(struct jump_entry *entry, 
	put_online_cpus();

}



void arch_jump_label_text_poke_early(jump_label_t addr)

{

	u32 *insn_p = (u32 *) (unsigned long) addr;



	*insn_p = 0x01000000;

	flushi(insn_p);

}



#endif

arch/x86/kernel/jump_label.c

+14 −6
Original line number Diff line number Diff line
@@ -24,8 +24,9 @@ union jump_code_union { 
	} __attribute__((packed));

};



void arch_jump_label_transform(struct jump_entry *entry,

			       enum jump_label_type type)

static void __jump_label_transform(struct jump_entry *entry,

				   enum jump_label_type type,

				   void *(*poker)(void *, const void *, size_t))

{

	union jump_code_union code;
@@ -35,17 +36,24 @@ void arch_jump_label_transform(struct jump_entry *entry, 
				(entry->code + JUMP_LABEL_NOP_SIZE);

	} else

		memcpy(&code, ideal_nops[NOP_ATOMIC5], JUMP_LABEL_NOP_SIZE);



	(*poker)((void *)entry->code, &code, JUMP_LABEL_NOP_SIZE);

}



void arch_jump_label_transform(struct jump_entry *entry,

			       enum jump_label_type type)

{

	get_online_cpus();

	mutex_lock(&text_mutex);

	text_poke_smp((void *)entry->code, &code, JUMP_LABEL_NOP_SIZE);

	__jump_label_transform(entry, type, text_poke_smp);

	mutex_unlock(&text_mutex);

	put_online_cpus();

}



void __init_or_module arch_jump_label_text_poke_early(jump_label_t addr)

void arch_jump_label_transform_static(struct jump_entry *entry,

				      enum jump_label_type type)

{

	text_poke_early((void *)addr, ideal_nops[NOP_ATOMIC5],

			JUMP_LABEL_NOP_SIZE);

	__jump_label_transform(entry, type, text_poke_early);

}



#endif

include/linux/jump_label.h

+14 −9
Original line number Diff line number Diff line
@@ -16,7 +16,7 @@ struct jump_label_key { 


# include <asm/jump_label.h>

# define HAVE_JUMP_LABEL

#endif

#endif	/* CC_HAVE_ASM_GOTO && CONFIG_JUMP_LABEL */



enum jump_label_type {

	JUMP_LABEL_DISABLE = 0,
@@ -28,9 +28,9 @@ struct module; 
#ifdef HAVE_JUMP_LABEL



#ifdef CONFIG_MODULES

#define JUMP_LABEL_INIT {{ 0 }, NULL, NULL}

#define JUMP_LABEL_INIT {ATOMIC_INIT(0), NULL, NULL}

#else

#define JUMP_LABEL_INIT {{ 0 }, NULL}

#define JUMP_LABEL_INIT {ATOMIC_INIT(0), NULL}

#endif



static __always_inline bool static_branch(struct jump_label_key *key)
@@ -41,18 +41,20 @@ static __always_inline bool static_branch(struct jump_label_key *key) 
extern struct jump_entry __start___jump_table[];

extern struct jump_entry __stop___jump_table[];



extern void jump_label_init(void);

extern void jump_label_lock(void);

extern void jump_label_unlock(void);

extern void arch_jump_label_transform(struct jump_entry *entry,

				      enum jump_label_type type);

extern void arch_jump_label_text_poke_early(jump_label_t addr);

extern void arch_jump_label_transform_static(struct jump_entry *entry,

					     enum jump_label_type type);

extern int jump_label_text_reserved(void *start, void *end);

extern void jump_label_inc(struct jump_label_key *key);

extern void jump_label_dec(struct jump_label_key *key);

extern bool jump_label_enabled(struct jump_label_key *key);

extern void jump_label_apply_nops(struct module *mod);



#else

#else  /* !HAVE_JUMP_LABEL */



#include <linux/atomic.h>
@@ -62,6 +64,10 @@ struct jump_label_key { 
	atomic_t enabled;

};



static __always_inline void jump_label_init(void)

{

}



static __always_inline bool static_branch(struct jump_label_key *key)

{

	if (unlikely(atomic_read(&key->enabled)))
@@ -96,7 +102,6 @@ static inline int jump_label_apply_nops(struct module *mod) 
{

	return 0;

}

#endif	/* HAVE_JUMP_LABEL */



#endif



#endif
 
#endif	/* _LINUX_JUMP_LABEL_H */

init/main.c

+3 −0
Original line number Diff line number Diff line
@@ -512,6 +512,9 @@ asmlinkage void __init start_kernel(void) 
	parse_args("Booting kernel", static_command_line, __start___param,

		   __stop___param - __start___param,

		   &unknown_bootoption);



	jump_label_init();



	/*

	 * These use large bootmem allocations and must precede

	 * kmem_cache_init()

CRA Git | Maintained and supported by SUSTech CRA and CCSE