Commit 965dd3ad authored by Nicholas Piggin's avatar Nicholas Piggin Committed by Michael Ellerman
Browse files

powerpc/64/syscall: Remove non-volatile GPR save optimisation



powerpc has an optimisation where interrupts avoid saving the
non-volatile (or callee saved) registers to the interrupt stack frame
if they are not required.

Two problems with this are that an interrupt does not always know
whether it will need non-volatiles; and if it does need them, they can
only be saved from the entry-scoped asm code (because we don't control
what the C compiler does with these registers).

system calls are the most difficult: some system calls always require
all registers (e.g., fork, to copy regs into the child). Sometimes
registers are only required under certain conditions (e.g., tracing,
signal delivery). These cases require ugly logic in the call
chains (e.g., ppc_fork), and require a lot of logic to be implemented
in asm.

So remove the optimisation for system calls, and always save NVGPRs on
entry. Modern high performance CPUs are not so sensitive, because the
stores are dense in cache and can be hidden by other expensive work in
the syscall path -- the null syscall selftests benchmark on POWER9 is
not slowed (124.40ns before and 123.64ns after, i.e., within the
noise).

Other interrupts retain the NVGPR optimisation for now.

Signed-off-by: default avatarNicholas Piggin <npiggin@gmail.com>
Signed-off-by: default avatarMichael Ellerman <mpe@ellerman.id.au>
Link: https://lore.kernel.org/r/20200225173541.1549955-24-npiggin@gmail.com
parent 71c3b05a
Loading
Loading
Loading
Loading
+13 −59
Original line number Diff line number Diff line
@@ -98,13 +98,14 @@ END_BTB_FLUSH_SECTION
	std	r11,_XER(r1)
	std	r11,_CTR(r1)
	std	r9,GPR13(r1)
	SAVE_NVGPRS(r1)
	mflr	r10
	/*
	 * This clears CR0.SO (bit 28), which is the error indication on
	 * return from this system call.
	 */
	rldimi	r2,r11,28,(63-28)
	li	r11,0xc01
	li	r11,0xc00
	std	r10,_LINK(r1)
	std	r11,_TRAP(r1)
	std	r3,ORIG_GPR3(r1)
@@ -323,7 +324,6 @@ END_FTR_SECTION_IFSET(CPU_FTR_HAS_PPR)

/* Traced system call support */
.Lsyscall_dotrace:
	bl	save_nvgprs
	addi	r3,r1,STACK_FRAME_OVERHEAD
	bl	do_syscall_trace_enter

@@ -408,7 +408,6 @@ END_FTR_SECTION_IFSET(CPU_FTR_HAS_PPR)
	mtmsrd	r10,1
#endif /* CONFIG_PPC_BOOK3E */

	bl	save_nvgprs
	addi	r3,r1,STACK_FRAME_OVERHEAD
	bl	do_syscall_trace_leave
	b	ret_from_except
@@ -442,62 +441,6 @@ END_FTR_SECTION_IFSET(CPU_FTR_HAS_PPR)
_ASM_NOKPROBE_SYMBOL(system_call_common);
_ASM_NOKPROBE_SYMBOL(system_call_exit);

/* Save non-volatile GPRs, if not already saved. */
_GLOBAL(save_nvgprs)
	ld	r11,_TRAP(r1)
	andi.	r0,r11,1
	beqlr-
	SAVE_NVGPRS(r1)
	clrrdi	r0,r11,1
	std	r0,_TRAP(r1)
	blr
_ASM_NOKPROBE_SYMBOL(save_nvgprs);

	
/*
 * The sigsuspend and rt_sigsuspend system calls can call do_signal
 * and thus put the process into the stopped state where we might
 * want to examine its user state with ptrace.  Therefore we need
 * to save all the nonvolatile registers (r14 - r31) before calling
 * the C code.  Similarly, fork, vfork and clone need the full
 * register state on the stack so that it can be copied to the child.
 */

_GLOBAL(ppc_fork)
	bl	save_nvgprs
	bl	sys_fork
	b	.Lsyscall_exit

_GLOBAL(ppc_vfork)
	bl	save_nvgprs
	bl	sys_vfork
	b	.Lsyscall_exit

_GLOBAL(ppc_clone)
	bl	save_nvgprs
	bl	sys_clone
	b	.Lsyscall_exit

_GLOBAL(ppc_clone3)
       bl      save_nvgprs
       bl      sys_clone3
       b       .Lsyscall_exit

_GLOBAL(ppc32_swapcontext)
	bl	save_nvgprs
	bl	compat_sys_swapcontext
	b	.Lsyscall_exit

_GLOBAL(ppc64_swapcontext)
	bl	save_nvgprs
	bl	sys_swapcontext
	b	.Lsyscall_exit

_GLOBAL(ppc_switch_endian)
	bl	save_nvgprs
	bl	sys_switch_endian
	b	.Lsyscall_exit

_GLOBAL(ret_from_fork)
	bl	schedule_tail
	REST_NVGPRS(r1)
@@ -516,6 +459,17 @@ _GLOBAL(ret_from_kernel_thread)
	li	r3,0
	b	.Lsyscall_exit

/* Save non-volatile GPRs, if not already saved. */
_GLOBAL(save_nvgprs)
	ld	r11,_TRAP(r1)
	andi.	r0,r11,1
	beqlr-
	SAVE_NVGPRS(r1)
	clrrdi	r0,r11,1
	std	r0,_TRAP(r1)
	blr
_ASM_NOKPROBE_SYMBOL(save_nvgprs);

#ifdef CONFIG_PPC_BOOK3S_64

#define FLUSH_COUNT_CACHE	\
+15 −7
Original line number Diff line number Diff line
@@ -9,7 +9,9 @@
#
0	nospu	restart_syscall			sys_restart_syscall
1	nospu	exit				sys_exit
2	nospu	fork				ppc_fork
2	32	fork				ppc_fork			sys_fork
2	64	fork				sys_fork
2	spu	fork				sys_ni_syscall
3	common	read				sys_read
4	common	write				sys_write
5	common	open				sys_open			compat_sys_open
@@ -158,7 +160,9 @@
119	32	sigreturn			sys_sigreturn			compat_sys_sigreturn
119	64	sigreturn			sys_ni_syscall
119	spu	sigreturn			sys_ni_syscall
120	nospu	clone				ppc_clone
120	32	clone				ppc_clone			sys_clone
120	64	clone				sys_clone
120	spu	clone				sys_ni_syscall
121	common	setdomainname			sys_setdomainname
122	common	uname				sys_newuname
123	common	modify_ldt			sys_ni_syscall
@@ -240,7 +244,9 @@
186	spu	sendfile			sys_sendfile64
187	common	getpmsg				sys_ni_syscall
188	common 	putpmsg				sys_ni_syscall
189	nospu	vfork				ppc_vfork
189	32	vfork				ppc_vfork			sys_vfork
189	64	vfork				sys_vfork
189	spu	vfork				sys_ni_syscall
190	common	ugetrlimit			sys_getrlimit			compat_sys_getrlimit
191	common	readahead			sys_readahead			compat_sys_readahead
192	32	mmap2				sys_mmap2			compat_sys_mmap2
@@ -316,8 +322,8 @@
248	32	clock_nanosleep			sys_clock_nanosleep_time32
248	64	clock_nanosleep			sys_clock_nanosleep
248	spu	clock_nanosleep			sys_clock_nanosleep
249	32	swapcontext			ppc_swapcontext			ppc32_swapcontext
249	64	swapcontext			ppc64_swapcontext
249	32	swapcontext			ppc_swapcontext			compat_sys_swapcontext
249	64	swapcontext			sys_swapcontext
249	spu	swapcontext			sys_ni_syscall
250	common	tgkill				sys_tgkill
251	32	utimes				sys_utimes_time32
@@ -456,7 +462,7 @@
361	common	bpf				sys_bpf
362	nospu	execveat			sys_execveat			compat_sys_execveat
363	32	switch_endian			sys_ni_syscall
363	64	switch_endian			ppc_switch_endian
363	64	switch_endian			sys_switch_endian
363	spu	switch_endian			sys_ni_syscall
364	common	userfaultfd			sys_userfaultfd
365	common	membarrier			sys_membarrier
@@ -516,6 +522,8 @@
432	common	fsmount				sys_fsmount
433	common	fspick				sys_fspick
434	common	pidfd_open			sys_pidfd_open
435	nospu	clone3				ppc_clone3
435	32	clone3				ppc_clone3			sys_clone3
435	64	clone3				sys_clone3
435	spu	clone3				sys_ni_syscall
437	common	openat2				sys_openat2
438	common	pidfd_getfd			sys_pidfd_getfd