Commit 464d1a78 authored by Jeremy Fitzhardinge's avatar Jeremy Fitzhardinge Committed by Andi Kleen
Browse files

[PATCH] i386: Convert i386 PDA code to use %fs 



Convert the PDA code to use %fs rather than %gs as the segment for
per-processor data.  This is because some processors show a small but
measurable performance gain for reloading a NULL segment selector (as %fs
generally is in user-space) versus a non-NULL one (as %gs generally is).

On modern processors the difference is very small, perhaps undetectable.
Some old AMD "K6 3D+" processors are noticably slower when %fs is used
rather than %gs; I have no idea why this might be, but I think they're
sufficiently rare that it doesn't matter much.

This patch also fixes the math emulator, which had not been adjusted to
match the changed struct pt_regs.

[frederik.deweerdt@gmail.com: fixit with gdb]
[mingo@elte.hu: Fix KVM too]

Signed-off-by: default avatarJeremy Fitzhardinge <jeremy@xensource.com>
Signed-off-by: default avatarAndi Kleen <ak@suse.de>
Cc: Ian Campbell <Ian.Campbell@XenSource.com>
Acked-by: default avatarIngo Molnar <mingo@elte.hu>
Acked-by: default avatarZachary Amsden <zach@vmware.com>
Cc: Eric Dumazet <dada1@cosmosbay.com>
Signed-off-by: default avatarFrederik Deweerdt <frederik.deweerdt@gmail.com>
Signed-off-by: default avatarAndrew Morton <akpm@osdl.org>
parent 54413927

arch/i386/kernel/asm-offsets.c

+1 −1
Original line number Diff line number Diff line
@@ -72,7 +72,7 @@ void foo(void) 
	OFFSET(PT_EAX, pt_regs, eax);

	OFFSET(PT_DS,  pt_regs, xds);

	OFFSET(PT_ES,  pt_regs, xes);

	OFFSET(PT_GS,  pt_regs, xgs);

	OFFSET(PT_FS,  pt_regs, xfs);

	OFFSET(PT_ORIG_EAX, pt_regs, orig_eax);

	OFFSET(PT_EIP, pt_regs, eip);

	OFFSET(PT_CS,  pt_regs, xcs);

arch/i386/kernel/cpu/common.c

+7 −7
Original line number Diff line number Diff line
@@ -605,7 +605,7 @@ void __init early_cpu_init(void) 
struct pt_regs * __devinit idle_regs(struct pt_regs *regs)

{

	memset(regs, 0, sizeof(struct pt_regs));

	regs->xgs = __KERNEL_PDA;

	regs->xfs = __KERNEL_PDA;

	return regs;

}
@@ -662,12 +662,12 @@ struct i386_pda boot_pda = { 
	.pcurrent = &init_task,

};



static inline void set_kernel_gs(void)

static inline void set_kernel_fs(void)

{

	/* Set %gs for this CPU's PDA.  Memory clobber is to create a

	/* Set %fs for this CPU's PDA.  Memory clobber is to create a

	   barrier with respect to any PDA operations, so the compiler

	   doesn't move any before here. */

	asm volatile ("mov %0, %%gs" : : "r" (__KERNEL_PDA) : "memory");

	asm volatile ("mov %0, %%fs" : : "r" (__KERNEL_PDA) : "memory");

}



/* Initialize the CPU's GDT and PDA.  The boot CPU does this for
@@ -718,7 +718,7 @@ void __cpuinit cpu_set_gdt(int cpu) 
	   the boot CPU, this will transition from the boot gdt+pda to

	   the real ones). */

	load_gdt(cpu_gdt_descr);

	set_kernel_gs();

	set_kernel_fs();

}



/* Common CPU init for both boot and secondary CPUs */
@@ -764,8 +764,8 @@ static void __cpuinit _cpu_init(int cpu, struct task_struct *curr) 
	__set_tss_desc(cpu, GDT_ENTRY_DOUBLEFAULT_TSS, &doublefault_tss);

#endif



	/* Clear %fs. */

	asm volatile ("mov %0, %%fs" : : "r" (0));

	/* Clear %gs. */

	asm volatile ("mov %0, %%gs" : : "r" (0));



	/* Clear all 6 debug registers: */

	set_debugreg(0, 0);

arch/i386/kernel/entry.S

+16 −16
Original line number Diff line number Diff line
@@ -30,7 +30,7 @@ 
 *	18(%esp) - %eax

 *	1C(%esp) - %ds

 *	20(%esp) - %es

 *	24(%esp) - %gs

 *	24(%esp) - %fs

 *	28(%esp) - orig_eax

 *	2C(%esp) - %eip

 *	30(%esp) - %cs
@@ -99,9 +99,9 @@ VM_MASK = 0x00020000 


#define SAVE_ALL \

	cld; \

	pushl %gs; \

	pushl %fs; \

	CFI_ADJUST_CFA_OFFSET 4;\

	/*CFI_REL_OFFSET gs, 0;*/\

	/*CFI_REL_OFFSET fs, 0;*/\

	pushl %es; \

	CFI_ADJUST_CFA_OFFSET 4;\

	/*CFI_REL_OFFSET es, 0;*/\
@@ -133,7 +133,7 @@ VM_MASK = 0x00020000 
	movl %edx, %ds; \

	movl %edx, %es; \

	movl $(__KERNEL_PDA), %edx; \

	movl %edx, %gs

	movl %edx, %fs



#define RESTORE_INT_REGS \

	popl %ebx;	\
@@ -166,9 +166,9 @@ VM_MASK = 0x00020000 
2:	popl %es;	\

	CFI_ADJUST_CFA_OFFSET -4;\

	/*CFI_RESTORE es;*/\

3:	popl %gs;	\

3:	popl %fs;	\

	CFI_ADJUST_CFA_OFFSET -4;\

	/*CFI_RESTORE gs;*/\

	/*CFI_RESTORE fs;*/\

.pushsection .fixup,"ax";	\

4:	movl $0,(%esp);	\

	jmp 1b;		\
@@ -349,11 +349,11 @@ sysenter_past_esp: 
	movl PT_OLDESP(%esp), %ecx

	xorl %ebp,%ebp

	TRACE_IRQS_ON

1:	mov  PT_GS(%esp), %gs

1:	mov  PT_FS(%esp), %fs

	ENABLE_INTERRUPTS_SYSEXIT

	CFI_ENDPROC

.pushsection .fixup,"ax"

2:	movl $0,PT_GS(%esp)

2:	movl $0,PT_FS(%esp)

	jmp 1b

.section __ex_table,"a"

	.align 4
@@ -550,7 +550,7 @@ syscall_badsys: 


#define FIXUP_ESPFIX_STACK \

	/* since we are on a wrong stack, we cant make it a C code :( */ \

	movl %gs:PDA_cpu, %ebx; \

	movl %fs:PDA_cpu, %ebx; \

	PER_CPU(cpu_gdt_descr, %ebx); \

	movl GDS_address(%ebx), %ebx; \

	GET_DESC_BASE(GDT_ENTRY_ESPFIX_SS, %ebx, %eax, %ax, %al, %ah); \
@@ -632,7 +632,7 @@ KPROBE_ENTRY(page_fault) 
	CFI_ADJUST_CFA_OFFSET 4

	ALIGN

error_code:

	/* the function address is in %gs's slot on the stack */

	/* the function address is in %fs's slot on the stack */

	pushl %es

	CFI_ADJUST_CFA_OFFSET 4

	/*CFI_REL_OFFSET es, 0*/
@@ -661,20 +661,20 @@ error_code: 
	CFI_ADJUST_CFA_OFFSET 4

	CFI_REL_OFFSET ebx, 0

	cld

	pushl %gs

	pushl %fs

	CFI_ADJUST_CFA_OFFSET 4

	/*CFI_REL_OFFSET gs, 0*/

	/*CFI_REL_OFFSET fs, 0*/

	movl $(__KERNEL_PDA), %ecx

	movl %ecx, %gs

	movl %ecx, %fs

	UNWIND_ESPFIX_STACK

	popl %ecx

	CFI_ADJUST_CFA_OFFSET -4

	/*CFI_REGISTER es, ecx*/

	movl PT_GS(%esp), %edi		# get the function address

	movl PT_FS(%esp), %edi		# get the function address

	movl PT_ORIG_EAX(%esp), %edx	# get the error code

	movl $-1, PT_ORIG_EAX(%esp)	# no syscall to restart

	mov  %ecx, PT_GS(%esp)

	/*CFI_REL_OFFSET gs, ES*/

	mov  %ecx, PT_FS(%esp)

	/*CFI_REL_OFFSET fs, ES*/

	movl $(__USER_DS), %ecx

	movl %ecx, %ds

	movl %ecx, %es

arch/i386/kernel/head.S

+3 −3
Original line number Diff line number Diff line
@@ -319,12 +319,12 @@ is386: movl $2,%ecx # set MP 
	movl %eax,%ds

	movl %eax,%es



	xorl %eax,%eax			# Clear FS and LDT

	movl %eax,%fs

	xorl %eax,%eax			# Clear GS and LDT

	movl %eax,%gs

	lldt %ax



	movl $(__KERNEL_PDA),%eax

	mov  %eax,%gs

	mov  %eax,%fs



	cld			# gcc2 wants the direction flag cleared at all times

	pushl $0		# fake return address for unwinder

arch/i386/kernel/kprobes.c

+2 −2
Original line number Diff line number Diff line
@@ -363,7 +363,7 @@ no_kprobe: 
			"	pushf\n"

			/* skip cs, eip, orig_eax */

			"	subl $12, %esp\n"

			"	pushl %gs\n"

			"	pushl %fs\n"

			"	pushl %ds\n"

			"	pushl %es\n"

			"	pushl %eax\n"
@@ -387,7 +387,7 @@ no_kprobe: 
			"	popl %edi\n"

			"	popl %ebp\n"

			"	popl %eax\n"

			/* skip eip, orig_eax, es, ds, gs */

			/* skip eip, orig_eax, es, ds, fs */

			"	addl $20, %esp\n"

			"	popf\n"

			"	ret\n");

CRA Git | Maintained and supported by SUSTech CRA and CCSE