x86-64: Move kernelstack from PDA to per-cpu. (9af45651) · Commits · 戴 / test

arch/x86/ia32/ia32entry.S

+4 −4

Original line number	Diff line number	Diff line
		@@ -112,8 +112,8 @@ ENTRY(ia32_sysenter_target)
		CFI_DEF_CFA rsp,0
		CFI_REGISTER rsp,rbp
		SWAPGS_UNSAFE_STACK
		movq %gs:pda_kernelstack, %rsp
		addq $(PDA_STACKOFFSET),%rsp
		movq PER_CPU_VAR(kernel_stack), %rsp
		addq $(KERNEL_STACK_OFFSET),%rsp
		/*
		* No need to follow this irqs on/off section: the syscall
		* disabled irqs, here we enable it straight after entry:
		@@ -273,13 +273,13 @@ ENDPROC(ia32_sysenter_target)
		ENTRY(ia32_cstar_target)
		CFI_STARTPROC32 simple
		CFI_SIGNAL_FRAME
		CFI_DEF_CFA rsp,PDA_STACKOFFSET
		CFI_DEF_CFA rsp,KERNEL_STACK_OFFSET
		CFI_REGISTER rip,rcx
		/CFI_REGISTER rflags,r11/
		SWAPGS_UNSAFE_STACK
		movl %esp,%r8d
		CFI_REGISTER rsp,r8
		movq %gs:pda_kernelstack,%rsp
		movq PER_CPU_VAR(kernel_stack),%rsp
		/*
		* No need to follow this irqs on/off section: the syscall
		* disabled irqs and here we enable it straight after entry:

+1 −3

Original line number	Diff line number	Diff line
		@@ -13,7 +13,7 @@
		struct x8664_pda {
		unsigned long unused1;
		unsigned long unused2;
		unsigned long kernelstack; /* 16 top of kernel stack for current */
		unsigned long unused3;
		unsigned long oldrsp; /* 24 user rsp for system call */
		int irqcount; /* 32 Irq nesting counter. Starts -1 */
		unsigned int unused6; /* 36 was cpunumber */
		@@ -44,6 +44,4 @@ extern void pda_init(int);

		#endif

		#define PDA_STACKOFFSET (5*8)

		#endif /* _ASM_X86_PDA_H */

+8 −12

Original line number	Diff line number	Diff line
		@@ -194,25 +194,21 @@ static inline struct thread_info *current_thread_info(void)

		#else /* X86_32 */

		#include <asm/pda.h>
		#include <asm/percpu.h>
		#define KERNEL_STACK_OFFSET (5*8)

		/*
		* macros/functions for gaining access to the thread information structure
		* preempt_count needs to be 1 initially, until the scheduler is functional.
		*/
		#ifndef __ASSEMBLY__
		static inline struct thread_info *current_thread_info(void)
		{
		struct thread_info *ti;
		ti = (void *)(read_pda(kernelstack) + PDA_STACKOFFSET - THREAD_SIZE);
		return ti;
		}
		DECLARE_PER_CPU(unsigned long, kernel_stack);

		/* do not use in interrupt context */
		static inline struct thread_info *stack_thread_info(void)
		static inline struct thread_info *current_thread_info(void)
		{
		struct thread_info *ti;
		asm("andq %%rsp,%0; " : "=r" (ti) : "0" (~(THREAD_SIZE - 1)));
		ti = (void *)(percpu_read(kernel_stack) +
		KERNEL_STACK_OFFSET - THREAD_SIZE);
		return ti;
		}

		@@ -220,8 +216,8 @@ static inline struct thread_info *stack_thread_info(void)

		/* how to get the thread information struct from ASM */
		#define GET_THREAD_INFO(reg) \
		movq %gs:pda_kernelstack,reg ; \
		subq $(THREAD_SIZE-PDA_STACKOFFSET),reg
		movq PER_CPU_VAR(kernel_stack),reg ; \
		subq $(THREAD_SIZE-KERNEL_STACK_OFFSET),reg

		#endif

+0 −1

Original line number	Diff line number	Diff line
		@@ -49,7 +49,6 @@ int main(void)
		BLANK();
		#undef ENTRY
		#define ENTRY(entry) DEFINE(pda_ ## entry, offsetof(struct x8664_pda, entry))
		ENTRY(kernelstack);
		ENTRY(oldrsp);
		ENTRY(irqcount);
		DEFINE(pda_size, sizeof(struct x8664_pda));

+4 −2

Original line number	Diff line number	Diff line
		@@ -889,6 +889,10 @@ DEFINE_PER_CPU(char *, irq_stack_ptr) =
		per_cpu_var(irq_stack) + IRQ_STACK_SIZE - 64;
		#endif

		DEFINE_PER_CPU(unsigned long, kernel_stack) =
		(unsigned long)&init_thread_union - KERNEL_STACK_OFFSET + THREAD_SIZE;
		EXPORT_PER_CPU_SYMBOL(kernel_stack);

		void __cpuinit pda_init(int cpu)
		{
		struct x8664_pda *pda = cpu_pda(cpu);
		@@ -900,8 +904,6 @@ void __cpuinit pda_init(int cpu)
		load_pda_offset(cpu);

		pda->irqcount = -1;
		pda->kernelstack = (unsigned long)stack_thread_info() -
		PDA_STACKOFFSET + THREAD_SIZE;

		if (cpu != 0) {
		if (pda->nodenumber == 0 && cpu_to_node(cpu) != NUMA_NO_NODE)