Commit d167a518 authored by Gerd Hoffmann's avatar Gerd Hoffmann Committed by Linus Torvalds
Browse files

[PATCH] x86_64: x86_64 version of the smp alternative patch. 



Changes are largely identical to the i386 version:

 * alternative #define are moved to the new alternative.h file.
 * one new elf section with pointers to the lock prefixes which can be
   nop'ed out for non-smp.
 * two new elf sections simliar to the "classic" alternatives to
   replace SMP code with simpler UP code.
 * fixup headers to use alternative.h instead of defining their own
   LOCK / LOCK_PREFIX macros.

The patch reuses the i386 version of the alternatives code to avoid code
duplication.  The code in alternatives.c was shuffled around a bit to
reduce the number of #ifdefs needed.  It also got some tweaks needed for
x86_64 (vsyscall page handling) and new features (noreplacement option
which was x86_64 only up to now).  Debug printk's are changed from
compile-time to runtime.

Loosely based on a early version from Bastian Blank <waldi@debian.org>

Signed-off-by: default avatarGerd Hoffmann <kraxel@suse.de>
Signed-off-by: default avatarAndi Kleen <ak@suse.de>
Signed-off-by: default avatarLinus Torvalds <torvalds@osdl.org>
parent 240cd6a8

arch/i386/kernel/alternative.c

+85 −33
Original line number Diff line number Diff line
@@ -4,27 +4,41 @@ 
#include <asm/alternative.h>

#include <asm/sections.h>



#define DEBUG 0

#if DEBUG

# define DPRINTK(fmt, args...) printk(fmt, args)

#else

# define DPRINTK(fmt, args...)

#endif

static int no_replacement    = 0;

static int smp_alt_once      = 0;

static int debug_alternative = 0;



static int __init noreplacement_setup(char *s)

{

	no_replacement = 1;

	return 1;

}

static int __init bootonly(char *str)

{

	smp_alt_once = 1;

	return 1;

}

static int __init debug_alt(char *str)

{

	debug_alternative = 1;

	return 1;

}



__setup("noreplacement", noreplacement_setup);

__setup("smp-alt-boot", bootonly);

__setup("debug-alternative", debug_alt);



#define DPRINTK(fmt, args...) if (debug_alternative) \

	printk(KERN_DEBUG fmt, args)



#ifdef GENERIC_NOP1

/* Use inline assembly to define this because the nops are defined

   as inline assembly strings in the include files and we cannot

   get them easily into strings. */

asm("\t.data\nintelnops: "

	GENERIC_NOP1 GENERIC_NOP2 GENERIC_NOP3 GENERIC_NOP4 GENERIC_NOP5 GENERIC_NOP6

	GENERIC_NOP7 GENERIC_NOP8);

asm("\t.data\nk8nops: "

	K8_NOP1 K8_NOP2 K8_NOP3 K8_NOP4 K8_NOP5 K8_NOP6

	K8_NOP7 K8_NOP8);

asm("\t.data\nk7nops: "

	K7_NOP1 K7_NOP2 K7_NOP3 K7_NOP4 K7_NOP5 K7_NOP6

	K7_NOP7 K7_NOP8);



extern unsigned char intelnops[], k8nops[], k7nops[];

extern unsigned char intelnops[];

static unsigned char *intel_nops[ASM_NOP_MAX+1] = {

	NULL,

	intelnops,
@@ -36,6 +50,13 @@ static unsigned char *intel_nops[ASM_NOP_MAX+1] = { 
	intelnops + 1 + 2 + 3 + 4 + 5 + 6,

	intelnops + 1 + 2 + 3 + 4 + 5 + 6 + 7,

};

#endif



#ifdef K8_NOP1

asm("\t.data\nk8nops: "

	K8_NOP1 K8_NOP2 K8_NOP3 K8_NOP4 K8_NOP5 K8_NOP6

	K8_NOP7 K8_NOP8);

extern unsigned char k8nops[];

static unsigned char *k8_nops[ASM_NOP_MAX+1] = {

	NULL,

	k8nops,
@@ -47,6 +68,13 @@ static unsigned char *k8_nops[ASM_NOP_MAX+1] = { 
	k8nops + 1 + 2 + 3 + 4 + 5 + 6,

	k8nops + 1 + 2 + 3 + 4 + 5 + 6 + 7,

};

#endif



#ifdef K7_NOP1

asm("\t.data\nk7nops: "

	K7_NOP1 K7_NOP2 K7_NOP3 K7_NOP4 K7_NOP5 K7_NOP6

	K7_NOP7 K7_NOP8);

extern unsigned char k7nops[];

static unsigned char *k7_nops[ASM_NOP_MAX+1] = {

	NULL,

	k7nops,
@@ -58,6 +86,18 @@ static unsigned char *k7_nops[ASM_NOP_MAX+1] = { 
	k7nops + 1 + 2 + 3 + 4 + 5 + 6,

	k7nops + 1 + 2 + 3 + 4 + 5 + 6 + 7,

};

#endif



#ifdef CONFIG_X86_64



extern char __vsyscall_0;

static inline unsigned char** find_nop_table(void)

{

	return k8_nops;

}



#else /* CONFIG_X86_64 */



static struct nop {

	int cpuid;

	unsigned char **noptable;
@@ -67,14 +107,6 @@ static struct nop { 
	{ -1, NULL }

};





extern struct alt_instr __alt_instructions[], __alt_instructions_end[];

extern struct alt_instr __smp_alt_instructions[], __smp_alt_instructions_end[];

extern u8 *__smp_locks[], *__smp_locks_end[];



extern u8 __smp_alt_begin[], __smp_alt_end[];





static unsigned char** find_nop_table(void)

{

	unsigned char **noptable = intel_nops;
@@ -89,6 +121,14 @@ static unsigned char** find_nop_table(void) 
	return noptable;

}



#endif /* CONFIG_X86_64 */



extern struct alt_instr __alt_instructions[], __alt_instructions_end[];

extern struct alt_instr __smp_alt_instructions[], __smp_alt_instructions_end[];

extern u8 *__smp_locks[], *__smp_locks_end[];



extern u8 __smp_alt_begin[], __smp_alt_end[];



/* Replace instructions with better alternatives for this CPU type.

   This runs before SMP is initialized to avoid SMP problems with

   self modifying code. This implies that assymetric systems where
@@ -99,6 +139,7 @@ void apply_alternatives(struct alt_instr *start, struct alt_instr *end) 
{

	unsigned char **noptable = find_nop_table();

	struct alt_instr *a;

	u8 *instr;

	int diff, i, k;



	DPRINTK("%s: alt table %p -> %p\n", __FUNCTION__, start, end);
@@ -106,7 +147,16 @@ void apply_alternatives(struct alt_instr *start, struct alt_instr *end) 
		BUG_ON(a->replacementlen > a->instrlen);

		if (!boot_cpu_has(a->cpuid))

			continue;

		memcpy(a->instr, a->replacement, a->replacementlen);

		instr = a->instr;

#ifdef CONFIG_X86_64

		/* vsyscall code is not mapped yet. resolve it manually. */

		if (instr >= (u8 *)VSYSCALL_START && instr < (u8*)VSYSCALL_END) {

			instr = __va(instr - (u8*)VSYSCALL_START + (u8*)__pa_symbol(&__vsyscall_0));

			DPRINTK("%s: vsyscall fixup: %p => %p\n",

				__FUNCTION__, a->instr, instr);

		}

#endif

		memcpy(instr, a->replacement, a->replacementlen);

		diff = a->instrlen - a->replacementlen;

		/* Pad the rest with nops */

		for (i = a->replacementlen; diff > 0; diff -= k, i += k) {
@@ -186,14 +236,6 @@ struct smp_alt_module { 
static LIST_HEAD(smp_alt_modules);

static DEFINE_SPINLOCK(smp_alt);



static int smp_alt_once = 0;

static int __init bootonly(char *str)

{

	smp_alt_once = 1;

	return 1;

}

__setup("smp-alt-boot", bootonly);



void alternatives_smp_module_add(struct module *mod, char *name,

				 void *locks, void *locks_end,

				 void *text,  void *text_end)
@@ -201,6 +243,9 @@ void alternatives_smp_module_add(struct module *mod, char *name, 
	struct smp_alt_module *smp;

	unsigned long flags;



	if (no_replacement)

		return;



	if (smp_alt_once) {

		if (boot_cpu_has(X86_FEATURE_UP))

			alternatives_smp_unlock(locks, locks_end,
@@ -235,7 +280,7 @@ void alternatives_smp_module_del(struct module *mod) 
	struct smp_alt_module *item;

	unsigned long flags;



	if (smp_alt_once)

	if (no_replacement || smp_alt_once)

		return;



	spin_lock_irqsave(&smp_alt, flags);
@@ -256,7 +301,7 @@ void alternatives_smp_switch(int smp) 
	struct smp_alt_module *mod;

	unsigned long flags;



	if (smp_alt_once)

	if (no_replacement || smp_alt_once)

		return;

	BUG_ON(!smp && (num_online_cpus() > 1));
@@ -285,6 +330,13 @@ void alternatives_smp_switch(int smp) 


void __init alternative_instructions(void)

{

	if (no_replacement) {

		printk(KERN_INFO "(SMP-)alternatives turned off\n");

		free_init_pages("SMP alternatives",

				(unsigned long)__smp_alt_begin,

				(unsigned long)__smp_alt_end);

		return;

	}

	apply_alternatives(__alt_instructions, __alt_instructions_end);



	/* switch to patch-once-at-boottime-only mode and free the

arch/x86_64/kernel/Makefile

+3 −1
Original line number Diff line number Diff line
@@ -8,7 +8,7 @@ obj-y := process.o signal.o entry.o traps.o irq.o \ 
		ptrace.o time.o ioport.o ldt.o setup.o i8259.o sys_x86_64.o \

		x8664_ksyms.o i387.o syscall.o vsyscall.o \

		setup64.o bootflag.o e820.o reboot.o quirks.o i8237.o \

		pci-dma.o pci-nommu.o

		pci-dma.o pci-nommu.o alternative.o



obj-$(CONFIG_X86_MCE)         += mce.o

obj-$(CONFIG_X86_MCE_INTEL)	+= mce_intel.o
@@ -49,3 +49,5 @@ intel_cacheinfo-y += ../../i386/kernel/cpu/intel_cacheinfo.o 
quirks-y			+= ../../i386/kernel/quirks.o

i8237-y				+= ../../i386/kernel/i8237.o

msr-$(subst m,y,$(CONFIG_X86_MSR))  += ../../i386/kernel/msr.o

alternative-y			+= ../../i386/kernel/alternative.o

arch/x86_64/kernel/module.c

+25 −13
Original line number Diff line number Diff line
@@ -145,26 +145,38 @@ int apply_relocate(Elf_Shdr *sechdrs, 
	return -ENOSYS;

} 



extern void apply_alternatives(void *start, void *end); 



int module_finalize(const Elf_Ehdr *hdr,

                    const Elf_Shdr *sechdrs,

                    struct module *me)

{

	const Elf_Shdr *s;

	const Elf_Shdr *s, *text = NULL, *alt = NULL, *locks = NULL;

	char *secstrings = (void *)hdr + sechdrs[hdr->e_shstrndx].sh_offset;



	/* look for .altinstructions to patch */ 

	for (s = sechdrs; s < sechdrs + hdr->e_shnum; s++) {

		void *seg; 		

		if (strcmp(".altinstructions", secstrings + s->sh_name))

			continue;

		seg = (void *)s->sh_addr; 

		apply_alternatives(seg, seg + s->sh_size); 

		if (!strcmp(".text", secstrings + s->sh_name))

			text = s;

		if (!strcmp(".altinstructions", secstrings + s->sh_name))

			alt = s;

		if (!strcmp(".smp_locks", secstrings + s->sh_name))

			locks= s;

	}



	if (alt) {

		/* patch .altinstructions */

		void *aseg = (void *)alt->sh_addr;

		apply_alternatives(aseg, aseg + alt->sh_size);

	}

	if (locks && text) {

		void *lseg = (void *)locks->sh_addr;

		void *tseg = (void *)text->sh_addr;

		alternatives_smp_module_add(me, me->name,

					    lseg, lseg + locks->sh_size,

					    tseg, tseg + text->sh_size);

	}

	return 0;

}



void module_arch_cleanup(struct module *mod)

{

	alternatives_smp_module_del(mod);

}

arch/x86_64/kernel/setup.c

+1 −75
Original line number Diff line number Diff line
@@ -473,80 +473,6 @@ contig_initmem_init(unsigned long start_pfn, unsigned long end_pfn) 
} 

#endif



/* Use inline assembly to define this because the nops are defined 

   as inline assembly strings in the include files and we cannot 

   get them easily into strings. */

asm("\t.data\nk8nops: " 

    K8_NOP1 K8_NOP2 K8_NOP3 K8_NOP4 K8_NOP5 K8_NOP6

    K8_NOP7 K8_NOP8); 

    

extern unsigned char k8nops[];

static unsigned char *k8_nops[ASM_NOP_MAX+1] = { 

     NULL,

     k8nops,

     k8nops + 1,

     k8nops + 1 + 2,

     k8nops + 1 + 2 + 3,

     k8nops + 1 + 2 + 3 + 4,

     k8nops + 1 + 2 + 3 + 4 + 5,

     k8nops + 1 + 2 + 3 + 4 + 5 + 6,

     k8nops + 1 + 2 + 3 + 4 + 5 + 6 + 7,

}; 



extern char __vsyscall_0;



/* Replace instructions with better alternatives for this CPU type.



   This runs before SMP is initialized to avoid SMP problems with

   self modifying code. This implies that assymetric systems where

   APs have less capabilities than the boot processor are not handled. 

   In this case boot with "noreplacement". */ 

void apply_alternatives(void *start, void *end) 

{ 

	struct alt_instr *a; 

	int diff, i, k;

	for (a = start; (void *)a < end; a++) { 

		u8 *instr;



		if (!boot_cpu_has(a->cpuid))

			continue;



		BUG_ON(a->replacementlen > a->instrlen); 

		instr = a->instr;

		/* vsyscall code is not mapped yet. resolve it manually. */

		if (instr >= (u8 *)VSYSCALL_START && instr < (u8*)VSYSCALL_END)

			instr = __va(instr - (u8*)VSYSCALL_START + (u8*)__pa_symbol(&__vsyscall_0));

		__inline_memcpy(instr, a->replacement, a->replacementlen);

		diff = a->instrlen - a->replacementlen; 



		/* Pad the rest with nops */

		for (i = a->replacementlen; diff > 0; diff -= k, i += k) {

			k = diff;

			if (k > ASM_NOP_MAX)

				k = ASM_NOP_MAX;

			__inline_memcpy(instr + i, k8_nops[k], k);

		} 

	}

} 



static int no_replacement __initdata = 0; 

 

void __init alternative_instructions(void)

{

	extern struct alt_instr __alt_instructions[], __alt_instructions_end[];

	if (no_replacement) 

		return;

	apply_alternatives(__alt_instructions, __alt_instructions_end);

}



static int __init noreplacement_setup(char *s)

{ 

     no_replacement = 1; 

     return 1;

} 



__setup("noreplacement", noreplacement_setup); 



#if defined(CONFIG_EDD) || defined(CONFIG_EDD_MODULE)

struct edd edd;

#ifdef CONFIG_EDD_MODULE
@@ -1303,7 +1229,7 @@ static int show_cpuinfo(struct seq_file *m, void *v) 
		/* Other (Linux-defined) */

		"cxmmx", NULL, "cyrix_arr", "centaur_mcr", NULL,

		"constant_tsc", NULL, NULL,

		NULL, NULL, NULL, NULL, NULL, NULL, NULL, NULL,

		"up", NULL, NULL, NULL, NULL, NULL, NULL, NULL,

		NULL, NULL, NULL, NULL, NULL, NULL, NULL, NULL,

		NULL, NULL, NULL, NULL, NULL, NULL, NULL, NULL,

arch/x86_64/kernel/smpboot.c

+4 −0
Original line number Diff line number Diff line
@@ -797,6 +797,8 @@ static int __cpuinit do_boot_cpu(int cpu, int apicid) 
	}





	alternatives_smp_switch(1);



	c_idle.idle = get_idle_for_cpu(cpu);



	if (c_idle.idle) {
@@ -1259,6 +1261,8 @@ void __cpu_die(unsigned int cpu) 
		/* They ack this in play_dead by setting CPU_DEAD */

		if (per_cpu(cpu_state, cpu) == CPU_DEAD) {

			printk ("CPU %d is now offline\n", cpu);

			if (1 == num_online_cpus())

				alternatives_smp_switch(0);

			return;

		}

		msleep(100);

CRA Git | Maintained and supported by SUSTech CRA and CCSE