Commit dc7ee00d authored by Martin Schwidefsky's avatar Martin Schwidefsky
Browse files

s390: lowcore stack pointer offsets 



Store the stack pointers in the lowcore for the kernel stack, the async
stack and the panic stack with the offset required for the first user.
This avoids an unnecessary add instruction on the system call path.

Signed-off-by: default avatarMartin Schwidefsky <schwidefsky@de.ibm.com>
parent b8668fd0

arch/s390/kernel/entry.S

+16 −20
Original line number Diff line number Diff line
@@ -45,6 +45,7 @@ _TIF_TRACE = (_TIF_SYSCALL_TRACE | _TIF_SYSCALL_AUDIT | _TIF_SECCOMP | \ 


STACK_SHIFT = PAGE_SHIFT + THREAD_ORDER

STACK_SIZE  = 1 << STACK_SHIFT

STACK_INIT  = STACK_SIZE - STACK_FRAME_OVERHEAD - __PT_SIZE



#define BASED(name) name-system_call(%r13)
@@ -97,10 +98,10 @@ STACK_SIZE = 1 << STACK_SHIFT 
	sra	%r14,\shift

	jnz	1f

	CHECK_STACK 1<<\shift,\savearea

	ahi	%r15,-(STACK_FRAME_OVERHEAD + __PT_SIZE)

	j	2f

1:	l	%r15,\stack		# load target stack

2:	ahi	%r15,-(STACK_FRAME_OVERHEAD + __PT_SIZE)

	la	%r11,STACK_FRAME_OVERHEAD(%r15)

2:	la	%r11,STACK_FRAME_OVERHEAD(%r15)

	.endm



	.macro	ADD64 high,low,timer
@@ -150,7 +151,7 @@ ENTRY(__switch_to) 
	l	%r4,__THREAD_info(%r2)		# get thread_info of prev

	l	%r5,__THREAD_info(%r3)		# get thread_info of next

	lr	%r15,%r5

	ahi	%r15,STACK_SIZE			# end of kernel stack of next

	ahi	%r15,STACK_INIT			# end of kernel stack of next

	st	%r3,__LC_CURRENT		# store task struct of next

	st	%r5,__LC_THREAD_INFO		# store thread info of next

	st	%r15,__LC_KERNEL_STACK		# store end of kernel stack
@@ -178,7 +179,6 @@ sysc_stm: 
	l	%r13,__LC_SVC_NEW_PSW+4

sysc_per:

	l	%r15,__LC_KERNEL_STACK

	ahi	%r15,-(STACK_FRAME_OVERHEAD + __PT_SIZE)

	la	%r11,STACK_FRAME_OVERHEAD(%r15)	# pointer to pt_regs

sysc_vtime:

	UPDATE_VTIME %r8,%r9,__LC_SYNC_ENTER_TIMER
@@ -359,11 +359,11 @@ ENTRY(pgm_check_handler) 
	tm	__LC_PGM_ILC+3,0x80	# check for per exception

	jnz	pgm_svcper		# -> single stepped svc

0:	CHECK_STACK STACK_SIZE,__LC_SAVE_AREA_SYNC

	ahi	%r15,-(STACK_FRAME_OVERHEAD + __PT_SIZE)

	j	2f

1:	UPDATE_VTIME %r14,%r15,__LC_SYNC_ENTER_TIMER

	l	%r15,__LC_KERNEL_STACK

2:	ahi	%r15,-(STACK_FRAME_OVERHEAD + __PT_SIZE)

	la	%r11,STACK_FRAME_OVERHEAD(%r15)

2:	la	%r11,STACK_FRAME_OVERHEAD(%r15)

	stm	%r0,%r7,__PT_R0(%r11)

	mvc	__PT_R8(32,%r11),__LC_SAVE_AREA_SYNC

	stm	%r8,%r9,__PT_PSW(%r11)
@@ -485,7 +485,6 @@ io_work: 
#

io_work_user:

	l	%r1,__LC_KERNEL_STACK

	ahi	%r1,-(STACK_FRAME_OVERHEAD + __PT_SIZE)

	mvc	STACK_FRAME_OVERHEAD(__PT_SIZE,%r1),0(%r11)

	xc	__SF_BACKCHAIN(4,%r1),__SF_BACKCHAIN(%r1)

	la	%r11,STACK_FRAME_OVERHEAD(%r1)
@@ -646,7 +645,6 @@ mcck_skip: 
	tm	__PT_PSW+1(%r11),0x01	# returning to user ?

	jno	mcck_return

	l	%r1,__LC_KERNEL_STACK	# switch to kernel stack

	ahi	%r1,-(STACK_FRAME_OVERHEAD + __PT_SIZE)

	mvc	STACK_FRAME_OVERHEAD(__PT_SIZE,%r1),0(%r11)

	xc	__SF_BACKCHAIN(4,%r1),__SF_BACKCHAIN(%r1)

	la	%r11,STACK_FRAME_OVERHEAD(%r15)
@@ -674,6 +672,7 @@ mcck_panic: 
	sra	%r14,PAGE_SHIFT

	jz	0f

	l	%r15,__LC_PANIC_STACK

	j	mcck_skip

0:	ahi	%r15,-(STACK_FRAME_OVERHEAD + __PT_SIZE)

	j	mcck_skip
@@ -714,12 +713,10 @@ ENTRY(restart_int_handler) 
 */

stack_overflow:

	l	%r15,__LC_PANIC_STACK	# change to panic stack

	ahi	%r15,-__PT_SIZE		# create pt_regs

	stm	%r0,%r7,__PT_R0(%r15)

	stm	%r8,%r9,__PT_PSW(%r15)

	la	%r11,STACK_FRAME_OVERHEAD(%r15)

	stm	%r0,%r7,__PT_R0(%r11)

	stm	%r8,%r9,__PT_PSW(%r11)

	mvc	__PT_R8(32,%r11),0(%r14)

	lr	%r15,%r11

	ahi	%r15,-STACK_FRAME_OVERHEAD

	l	%r1,BASED(1f)

	xc	__SF_BACKCHAIN(4,%r15),__SF_BACKCHAIN(%r15)

	lr	%r2,%r11		# pass pointer to pt_regs
@@ -799,15 +796,14 @@ cleanup_system_call: 
	mvc	__LC_LAST_UPDATE_TIMER(8),__LC_SYNC_ENTER_TIMER

	# set up saved register 11

	l	%r15,__LC_KERNEL_STACK

	ahi	%r15,-__PT_SIZE

	st	%r15,12(%r11)		# r11 pt_regs pointer

	la	%r9,STACK_FRAME_OVERHEAD(%r15)

	st	%r9,12(%r11)		# r11 pt_regs pointer

	# fill pt_regs

	mvc	__PT_R8(32,%r15),__LC_SAVE_AREA_SYNC

	stm	%r0,%r7,__PT_R0(%r15)

	mvc	__PT_PSW(8,%r15),__LC_SVC_OLD_PSW

	mvc	__PT_INT_CODE(4,%r15),__LC_SVC_ILC

	mvc	__PT_R8(32,%r9),__LC_SAVE_AREA_SYNC

	stm	%r0,%r7,__PT_R0(%r9)

	mvc	__PT_PSW(8,%r9),__LC_SVC_OLD_PSW

	mvc	__PT_INT_CODE(4,%r9),__LC_SVC_ILC

	# setup saved register 15

	ahi	%r15,-STACK_FRAME_OVERHEAD

	st	%r15,28(%r11)		# r15 stack pointer

	# set new psw address and exit

	l	%r9,BASED(cleanup_table+4)	# sysc_do_svc + 0x80000000

arch/s390/kernel/entry64.S

+14 −20
Original line number Diff line number Diff line
@@ -39,6 +39,7 @@ __PT_R15 = __PT_GPRS + 120 


STACK_SHIFT = PAGE_SHIFT + THREAD_ORDER

STACK_SIZE  = 1 << STACK_SHIFT

STACK_INIT = STACK_SIZE - STACK_FRAME_OVERHEAD - __PT_SIZE



_TIF_WORK_SVC = (_TIF_SIGPENDING | _TIF_NOTIFY_RESUME | _TIF_NEED_RESCHED | \

		 _TIF_MCCK_PENDING | _TIF_PER_TRAP )
@@ -124,10 +125,10 @@ _TIF_EXIT_SIE = (_TIF_SIGPENDING | _TIF_NEED_RESCHED | _TIF_MCCK_PENDING) 
	srag	%r14,%r14,\shift

	jnz	1f

	CHECK_STACK 1<<\shift,\savearea

	aghi	%r15,-(STACK_FRAME_OVERHEAD + __PT_SIZE)

	j	2f

1:	lg	%r15,\stack		# load target stack

2:	aghi	%r15,-(STACK_FRAME_OVERHEAD + __PT_SIZE)

	la	%r11,STACK_FRAME_OVERHEAD(%r15)

2:	la	%r11,STACK_FRAME_OVERHEAD(%r15)

	.endm



	.macro UPDATE_VTIME scratch,enter_timer
@@ -177,7 +178,7 @@ ENTRY(__switch_to) 
	lg	%r4,__THREAD_info(%r2)		# get thread_info of prev

	lg	%r5,__THREAD_info(%r3)		# get thread_info of next

	lgr	%r15,%r5

	aghi	%r15,STACK_SIZE			# end of kernel stack of next

	aghi	%r15,STACK_INIT			# end of kernel stack of next

	stg	%r3,__LC_CURRENT		# store task struct of next

	stg	%r5,__LC_THREAD_INFO		# store thread info of next

	stg	%r15,__LC_KERNEL_STACK		# store end of kernel stack
@@ -203,10 +204,8 @@ sysc_stmg: 
	stmg	%r8,%r15,__LC_SAVE_AREA_SYNC

	lg	%r10,__LC_LAST_BREAK

	lg	%r12,__LC_THREAD_INFO

	larl	%r13,system_call

sysc_per:

	lg	%r15,__LC_KERNEL_STACK

	aghi	%r15,-(STACK_FRAME_OVERHEAD + __PT_SIZE)

	la	%r11,STACK_FRAME_OVERHEAD(%r15)	# pointer to pt_regs

sysc_vtime:

	UPDATE_VTIME %r13,__LC_SYNC_ENTER_TIMER
@@ -389,6 +388,7 @@ ENTRY(pgm_check_handler) 
	tm	__LC_PGM_ILC+3,0x80	# check for per exception

	jnz	pgm_svcper		# -> single stepped svc

0:	CHECK_STACK STACK_SIZE,__LC_SAVE_AREA_SYNC

	aghi	%r15,-(STACK_FRAME_OVERHEAD + __PT_SIZE)

	j	2f

1:	UPDATE_VTIME %r14,__LC_SYNC_ENTER_TIMER

	LAST_BREAK %r14
@@ -398,8 +398,7 @@ ENTRY(pgm_check_handler) 
	tm	__LC_PGM_ILC+2,0x02	# check for transaction abort

	jz	2f

	mvc	__THREAD_trap_tdb(256,%r14),0(%r13)

2:	aghi	%r15,-(STACK_FRAME_OVERHEAD + __PT_SIZE)

	la	%r11,STACK_FRAME_OVERHEAD(%r15)

2:	la	%r11,STACK_FRAME_OVERHEAD(%r15)

	stmg	%r0,%r7,__PT_R0(%r11)

	mvc	__PT_R8(64,%r11),__LC_SAVE_AREA_SYNC

	stmg	%r8,%r9,__PT_PSW(%r11)
@@ -526,7 +525,6 @@ io_work: 
#

io_work_user:

	lg	%r1,__LC_KERNEL_STACK

	aghi	%r1,-(STACK_FRAME_OVERHEAD + __PT_SIZE)

	mvc	STACK_FRAME_OVERHEAD(__PT_SIZE,%r1),0(%r11)

	xc	__SF_BACKCHAIN(8,%r1),__SF_BACKCHAIN(%r1)

	la	%r11,STACK_FRAME_OVERHEAD(%r1)
@@ -688,7 +686,6 @@ mcck_skip: 
	tm	__PT_PSW+1(%r11),0x01	# returning to user ?

	jno	mcck_return

	lg	%r1,__LC_KERNEL_STACK	# switch to kernel stack

	aghi	%r1,-(STACK_FRAME_OVERHEAD + __PT_SIZE)

	mvc	STACK_FRAME_OVERHEAD(__PT_SIZE,%r1),0(%r11)

	xc	__SF_BACKCHAIN(8,%r1),__SF_BACKCHAIN(%r1)

	la	%r11,STACK_FRAME_OVERHEAD(%r1)
@@ -755,14 +752,12 @@ ENTRY(restart_int_handler) 
 * Setup a pt_regs so that show_trace can provide a good call trace.

 */

stack_overflow:

	lg	%r11,__LC_PANIC_STACK	# change to panic stack

	aghi	%r11,-__PT_SIZE		# create pt_regs

	lg	%r15,__LC_PANIC_STACK	# change to panic stack

	la	%r11,STACK_FRAME_OVERHEAD(%r15)

	stmg	%r0,%r7,__PT_R0(%r11)

	stmg	%r8,%r9,__PT_PSW(%r11)

	mvc	__PT_R8(64,%r11),0(%r14)

	stg	%r10,__PT_ORIG_GPR2(%r11) # store last break to orig_gpr2

	lgr	%r15,%r11

	aghi	%r15,-STACK_FRAME_OVERHEAD

	xc	__SF_BACKCHAIN(8,%r15),__SF_BACKCHAIN(%r15)

	lgr	%r2,%r11		# pass pointer to pt_regs

	jg	kernel_stack_overflow
@@ -846,15 +841,14 @@ cleanup_system_call: 
	mvc	__TI_last_break(8,%r12),16(%r11)

0:	# set up saved register r11

	lg	%r15,__LC_KERNEL_STACK

	aghi	%r15,-__PT_SIZE

	stg	%r15,24(%r11)		# r11 pt_regs pointer

	la	%r9,STACK_FRAME_OVERHEAD(%r15)

	stg	%r9,24(%r11)		# r11 pt_regs pointer

	# fill pt_regs

	mvc	__PT_R8(64,%r15),__LC_SAVE_AREA_SYNC

	stmg	%r0,%r7,__PT_R0(%r15)

	mvc	__PT_PSW(16,%r15),__LC_SVC_OLD_PSW

	mvc	__PT_INT_CODE(4,%r15),__LC_SVC_ILC

	mvc	__PT_R8(64,%r9),__LC_SAVE_AREA_SYNC

	stmg	%r0,%r7,__PT_R0(%r9)

	mvc	__PT_PSW(16,%r9),__LC_SVC_OLD_PSW

	mvc	__PT_INT_CODE(4,%r9),__LC_SVC_ILC

	# setup saved register r15

	aghi	%r15,-STACK_FRAME_OVERHEAD

	stg	%r15,56(%r11)		# r15 stack pointer

	# set new psw address and exit

	larl	%r9,sysc_do_svc

arch/s390/kernel/setup.c

+6 −3
Original line number Diff line number Diff line
@@ -377,11 +377,14 @@ static void __init setup_lowcore(void) 
		PSW_MASK_DAT | PSW_MASK_MCHECK;

	lc->io_new_psw.addr = PSW_ADDR_AMODE | (unsigned long) io_int_handler;

	lc->clock_comparator = -1ULL;

	lc->kernel_stack = ((unsigned long) &init_thread_union) + THREAD_SIZE;

	lc->kernel_stack = ((unsigned long) &init_thread_union)

		+ THREAD_SIZE - STACK_FRAME_OVERHEAD - sizeof(struct pt_regs);

	lc->async_stack = (unsigned long)

		__alloc_bootmem(ASYNC_SIZE, ASYNC_SIZE, 0) + ASYNC_SIZE;

		__alloc_bootmem(ASYNC_SIZE, ASYNC_SIZE, 0)

		+ ASYNC_SIZE - STACK_FRAME_OVERHEAD - sizeof(struct pt_regs);

	lc->panic_stack = (unsigned long)

		__alloc_bootmem(PAGE_SIZE, PAGE_SIZE, 0) + PAGE_SIZE;

		__alloc_bootmem(PAGE_SIZE, PAGE_SIZE, 0)

		+ PAGE_SIZE - STACK_FRAME_OVERHEAD - sizeof(struct pt_regs);

	lc->current_task = (unsigned long) init_thread_union.thread_info.task;

	lc->thread_info = (unsigned long) &init_thread_union;

	lc->machine_flags = S390_lowcore.machine_flags;

arch/s390/kernel/smp.c

+10 −5
Original line number Diff line number Diff line
@@ -181,8 +181,10 @@ static int __cpuinit pcpu_alloc_lowcore(struct pcpu *pcpu, int cpu) 
	lc = pcpu->lowcore;

	memcpy(lc, &S390_lowcore, 512);

	memset((char *) lc + 512, 0, sizeof(*lc) - 512);

	lc->async_stack = pcpu->async_stack + ASYNC_SIZE;

	lc->panic_stack = pcpu->panic_stack + PAGE_SIZE;

	lc->async_stack = pcpu->async_stack + ASYNC_SIZE

		- STACK_FRAME_OVERHEAD - sizeof(struct pt_regs);

	lc->panic_stack = pcpu->panic_stack + PAGE_SIZE

		- STACK_FRAME_OVERHEAD - sizeof(struct pt_regs);

	lc->cpu_nr = cpu;

#ifndef CONFIG_64BIT

	if (MACHINE_HAS_IEEE) {
@@ -253,7 +255,8 @@ static void pcpu_attach_task(struct pcpu *pcpu, struct task_struct *tsk) 
	struct _lowcore *lc = pcpu->lowcore;

	struct thread_info *ti = task_thread_info(tsk);



	lc->kernel_stack = (unsigned long) task_stack_page(tsk) + THREAD_SIZE;

	lc->kernel_stack = (unsigned long) task_stack_page(tsk)

		+ THREAD_SIZE - STACK_FRAME_OVERHEAD - sizeof(struct pt_regs);

	lc->thread_info = (unsigned long) task_thread_info(tsk);

	lc->current_task = (unsigned long) tsk;

	lc->user_timer = ti->user_timer;
@@ -810,8 +813,10 @@ void __init smp_prepare_boot_cpu(void) 
	pcpu->state = CPU_STATE_CONFIGURED;

	pcpu->address = boot_cpu_address;

	pcpu->lowcore = (struct _lowcore *)(unsigned long) store_prefix();

	pcpu->async_stack = S390_lowcore.async_stack - ASYNC_SIZE;

	pcpu->panic_stack = S390_lowcore.panic_stack - PAGE_SIZE;

	pcpu->async_stack = S390_lowcore.async_stack - ASYNC_SIZE

		+ STACK_FRAME_OVERHEAD + sizeof(struct pt_regs);

	pcpu->panic_stack = S390_lowcore.panic_stack - PAGE_SIZE

		+ STACK_FRAME_OVERHEAD + sizeof(struct pt_regs);

	S390_lowcore.percpu_offset = __per_cpu_offset[0];

	smp_cpu_set_polarization(0, POLARIZATION_UNKNOWN);

	set_cpu_present(0, true);

CRA Git | Maintained and supported by SUSTech CRA and CCSE