Commit 17968fbb authored by Anton Blanchard's avatar Anton Blanchard Committed by Benjamin Herrenschmidt
Browse files

powerpc: 64bit optimised __clear_user 

I noticed __clear_user high up in a profile of one of my RAID stress
tests. The testcase was doing a dd from /dev/zero which ends up
calling __clear_user.

__clear_user is basically a loop with a single 4 byte store which
is horribly slow. We can do much better by aligning the desination
and doing 32 bytes of 8 byte stores in a loop.

The following testcase was used to verify the patch:

http://ozlabs.org/~anton/junkcode/stress_clear_user.c



To show the improvement in performance I ran a dd from /dev/zero
to /dev/null on a POWER7 box:

Before:

# dd if=/dev/zero of=/dev/null bs=1M count=10000
10485760000 bytes (10 GB) copied, 3.72379 s, 2.8 GB/s

After:

# time dd if=/dev/zero of=/dev/null bs=1M count=10000
10485760000 bytes (10 GB) copied, 0.728318 s, 14.4 GB/s

Over 5x faster.

Signed-off-by: default avatarAnton Blanchard <anton@samba.org>
Signed-off-by: default avatarBenjamin Herrenschmidt <benh@kernel.crashing.org>
parent d136e273

arch/powerpc/lib/Makefile

+1 −1
Original line number Diff line number Diff line
@@ -17,7 +17,7 @@ obj-$(CONFIG_HAS_IOMEM) += devres.o 
obj-$(CONFIG_PPC64)	+= copypage_64.o copyuser_64.o \

			   memcpy_64.o usercopy_64.o mem_64.o string.o \

			   checksum_wrappers_64.o hweight_64.o \

			   copyuser_power7.o

			   copyuser_power7.o string_64.o

obj-$(CONFIG_XMON)	+= sstep.o ldstfp.o

obj-$(CONFIG_KPROBES)	+= sstep.o ldstfp.o

obj-$(CONFIG_HAVE_HW_BREAKPOINT)	+= sstep.o ldstfp.o

arch/powerpc/lib/string.S

+2 −0
Original line number Diff line number Diff line
@@ -119,6 +119,7 @@ _GLOBAL(memchr) 
2:	li	r3,0

	blr



#ifdef CONFIG_PPC32

_GLOBAL(__clear_user)

	addi	r6,r3,-4

	li	r3,0
@@ -160,3 +161,4 @@ _GLOBAL(__clear_user) 
	PPC_LONG	1b,91b

	PPC_LONG	8b,92b

	.text

#endif

arch/powerpc/lib/string_64.S

0 → 100644
+141 −0
Original line number Diff line number Diff line 
/*

 * This program is free software; you can redistribute it and/or modify

 * it under the terms of the GNU General Public License as published by

 * the Free Software Foundation; either version 2 of the License, or

 * (at your option) any later version.

 *

 * This program is distributed in the hope that it will be useful,

 * but WITHOUT ANY WARRANTY; without even the implied warranty of

 * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the

 * GNU General Public License for more details.

 *

 * You should have received a copy of the GNU General Public License

 * along with this program; if not, write to the Free Software

 * Foundation, Inc., 59 Temple Place - Suite 330, Boston, MA 02111-1307, USA.

 *

 * Copyright (C) IBM Corporation, 2012

 *

 * Author: Anton Blanchard <anton@au.ibm.com>

 */



#include <asm/ppc_asm.h>



/**

 * __clear_user: - Zero a block of memory in user space, with less checking.

 * @to:   Destination address, in user space.

 * @n:    Number of bytes to zero.

 *

 * Zero a block of memory in user space.  Caller must check

 * the specified block with access_ok() before calling this function.

 *

 * Returns number of bytes that could not be cleared.

 * On success, this will be zero.

 */



	.macro err1

100:

	.section __ex_table,"a"

	.align 3

	.llong 100b,.Ldo_err1

	.previous

	.endm



	.macro err2

200:

	.section __ex_table,"a"

	.align 3

	.llong 200b,.Ldo_err2

	.previous

	.endm



	.macro err3

300:

	.section __ex_table,"a"

	.align 3

	.llong 300b,.Ldo_err3

	.previous

	.endm



.Ldo_err1:

	mr	r3,r8



.Ldo_err2:

	mtctr	r4

1:

err3;	stb	r0,0(r3)

	addi	r3,r3,1

	addi	r4,r4,-1

	bdnz	1b



.Ldo_err3:

	mr	r3,r4

	blr



_GLOBAL(__clear_user)

	cmpdi	r4,32

	neg	r6,r3

	li	r0,0

	blt	.Lshort_clear

	mr	r8,r3

	mtocrf	0x01,r6

	clrldi	r6,r6,(64-3)



	/* Get the destination 8 byte aligned */

	bf	cr7*4+3,1f

err1;	stb	r0,0(r3)

	addi	r3,r3,1



1:	bf	cr7*4+2,2f

err1;	sth	r0,0(r3)

	addi	r3,r3,2



2:	bf	cr7*4+1,3f

err1;	stw	r0,0(r3)

	addi	r3,r3,4



3:	sub	r4,r4,r6

	srdi	r6,r4,5

	cmpdi	r4,32

	blt	.Lshort_clear

	mtctr	r6



	/* Do 32 byte chunks */

4:

err2;	std	r0,0(r3)

err2;	std	r0,8(r3)

err2;	std	r0,16(r3)

err2;	std	r0,24(r3)

	addi	r3,r3,32

	addi	r4,r4,-32

	bdnz	4b



.Lshort_clear:

	/* up to 31 bytes to go */

	cmpdi	r4,16

	blt	6f

err2;	std	r0,0(r3)

err2;	std	r0,8(r3)

	addi	r3,r3,16

	addi	r4,r4,-16



	/* Up to 15 bytes to go */

6:	mr	r8,r3

	clrldi	r4,r4,(64-4)

	mtocrf	0x01,r4

	bf	cr7*4+0,7f

err1;	std	r0,0(r3)

	addi	r3,r3,8



7:	bf	cr7*4+1,8f

err1;	stw	r0,0(r3)

	addi	r3,r3,4



8:	bf	cr7*4+2,9f

err1;	sth	r0,0(r3)

	addi	r3,r3,2



9:	bf	cr7*4+3,10f

err1;	stb	r0,0(r3)



10:	li	r3,0

	blr

CRA Git | Maintained and supported by SUSTech CRA and CCSE