ARM: 8812/1: Optimise copy_{from/to}_user for !CPU_USE_DOMAINS (f441882a) · Commits · 戴 / test

ARMv6+ processors do not use CONFIG_CPU_USE_DOMAINS and use privileged ldr/str instructions in copy_{from/to}_user. They are currently unnecessarily using single ldr/str instructions and can use ldm/stm instructions instead like memcpy does (but with appropriate fixup tables). This speeds up a "dd if=foo of=bar bs=32k" on a tmpfs filesystem by about 4% on my Cortex-A9. before:134217728 bytes (128.0MB) copied, 0.543848 seconds, 235.4MB/s before:134217728 bytes (128.0MB) copied, 0.538610 seconds, 237.6MB/s before:134217728 bytes (128.0MB) copied, 0.544356 seconds, 235.1MB/s before:134217728 bytes (128.0MB) copied, 0.544364 seconds, 235.1MB/s before:134217728 bytes (128.0MB) copied, 0.537130 seconds, 238.3MB/s before:134217728 bytes (128.0MB) copied, 0.533443 seconds, 240.0MB/s before:134217728 bytes (128.0MB) copied, 0.545691 seconds, 234.6MB/s before:134217728 bytes (128.0MB) copied, 0.534695 seconds, 239.4MB/s before:134217728 bytes (128.0MB) copied, 0.540561 seconds, 236.8MB/s before:134217728 bytes (128.0MB) copied, 0.541025 seconds, 236.6MB/s after:134217728 bytes (128.0MB) copied, 0.520445 seconds, 245.9MB/s after:134217728 bytes (128.0MB) copied, 0.527846 seconds, 242.5MB/s after:134217728 bytes (128.0MB) copied, 0.519510 seconds, 246.4MB/s after:134217728 bytes (128.0MB) copied, 0.527231 seconds, 242.8MB/s after:134217728 bytes (128.0MB) copied, 0.525030 seconds, 243.8MB/s after:134217728 bytes (128.0MB) copied, 0.524236 seconds, 244.2MB/s after:134217728 bytes (128.0MB) copied, 0.523659 seconds, 244.4MB/s after:134217728 bytes (128.0MB) copied, 0.525018 seconds, 243.8MB/s after:134217728 bytes (128.0MB) copied, 0.519249 seconds, 246.5MB/s after:134217728 bytes (128.0MB) copied, 0.518527 seconds, 246.9MB/s Reviewed-by:

Nicolas Pitre <nico@linaro.org> Signed-off-by:

Vincent Whitchurch <vincent.whitchurch@axis.com> Signed-off-by:

Russell King <rmk+kernel@armlinux.org.uk>

arch/arm/include/asm/assembler.h

+4 −2

Original line number	Diff line number	Diff line
		@@ -243,13 +243,15 @@
		.endm
		#endif

		#define USER(x...) \
		#define USERL(l, x...) \
		9999: x; \
		.pushsection __ex_table,"a"; \
		.align 3; \
		.long 9999b,9001f; \
		.long 9999b,l; \
		.popsection

		#define USER(x...) USERL(9001f, x)

		#ifdef CONFIG_SMP
		#define ALT_SMP(instr...) \
		9998: instr

arch/arm/lib/copy_from_user.S

+22 −1

Original line number	Diff line number	Diff line
		@@ -34,12 +34,13 @@
		* Number of bytes NOT copied.
		*/

		#ifdef CONFIG_CPU_USE_DOMAINS

		#ifndef CONFIG_THUMB2_KERNEL
		#define LDR1W_SHIFT 0
		#else
		#define LDR1W_SHIFT 1
		#endif
		#define STR1W_SHIFT 0

		.macro ldr1w ptr reg abort
		ldrusr \reg, \ptr, 4, abort=\abort
		@@ -57,10 +58,30 @@
		ldr4w \ptr, \reg5, \reg6, \reg7, \reg8, \abort
		.endm

		#else

		#define LDR1W_SHIFT 0

		.macro ldr1w ptr reg abort
		USERL(\abort, W(ldr) \reg, [\ptr], #4)
		.endm

		.macro ldr4w ptr reg1 reg2 reg3 reg4 abort
		USERL(\abort, ldmia \ptr!, {\reg1, \reg2, \reg3, \reg4})
		.endm

		.macro ldr8w ptr reg1 reg2 reg3 reg4 reg5 reg6 reg7 reg8 abort
		USERL(\abort, ldmia \ptr!, {\reg1, \reg2, \reg3, \reg4, \reg5, \reg6, \reg7, \reg8})
		.endm

		#endif /* CONFIG_CPU_USE_DOMAINS */

		.macro ldr1b ptr reg cond=al abort
		ldrusr \reg, \ptr, 1, \cond, abort=\abort
		.endm

		#define STR1W_SHIFT 0

		.macro str1w ptr reg abort
		W(str) \reg, [\ptr], #4
		.endm

arch/arm/lib/copy_to_user.S

+22 −5

Original line number	Diff line number	Diff line
		@@ -35,11 +35,6 @@
		*/

		#define LDR1W_SHIFT 0
		#ifndef CONFIG_THUMB2_KERNEL
		#define STR1W_SHIFT 0
		#else
		#define STR1W_SHIFT 1
		#endif

		.macro ldr1w ptr reg abort
		W(ldr) \reg, [\ptr], #4
		@@ -57,6 +52,14 @@
		ldr\cond\()b \reg, [\ptr], #1
		.endm

		#ifdef CONFIG_CPU_USE_DOMAINS

		#ifndef CONFIG_THUMB2_KERNEL
		#define STR1W_SHIFT 0
		#else
		#define STR1W_SHIFT 1
		#endif

		.macro str1w ptr reg abort
		strusr \reg, \ptr, 4, abort=\abort
		.endm
		@@ -72,6 +75,20 @@
		str1w \ptr, \reg8, \abort
		.endm

		#else

		#define STR1W_SHIFT 0

		.macro str1w ptr reg abort
		USERL(\abort, W(str) \reg, [\ptr], #4)
		.endm

		.macro str8w ptr reg1 reg2 reg3 reg4 reg5 reg6 reg7 reg8 abort
		USERL(\abort, stmia \ptr!, {\reg1, \reg2, \reg3, \reg4, \reg5, \reg6, \reg7, \reg8})
		.endm

		#endif /* CONFIG_CPU_USE_DOMAINS */

		.macro str1b ptr reg cond=al abort
		strusr \reg, \ptr, 1, \cond, abort=\abort
		.endm

Admin message