Commit 7269e881 authored by Jan Beulich's avatar Jan Beulich Committed by Ingo Molnar
Browse files

x86-64: Modify memcpy()/memset() alternatives mechanism 



In order to avoid unnecessary chains of branches, rather than
implementing memcpy()/memset()'s access to their alternative
implementations via a jump, patch the (larger) original function
directly.

The memcpy() part of this is slightly subtle: while alternative
instruction patching does itself use memcpy(), with the
replacement block being less than 64-bytes in size the main loop
of the original function doesn't get used for copying memcpy_c()
over memcpy(), and hence we can safely write over its beginning.

Also note that the CFI annotations are fine for both variants of
each of the functions.

Signed-off-by: default avatarJan Beulich <jbeulich@novell.com>
Cc: Nick Piggin <npiggin@suse.de>
Cc: Linus Torvalds <torvalds@linux-foundation.org>
Cc: Andrew Morton <akpm@linux-foundation.org>
LKML-Reference: <4B2BB8D30200007800026AF2@vpn.id2.novell.com>
Signed-off-by: default avatarIngo Molnar <mingo@elte.hu>
parent 1b1d9258

arch/x86/lib/memcpy_64.S

+8 −15
Original line number Diff line number Diff line
@@ -20,12 +20,11 @@ 
/*

 * memcpy_c() - fast string ops (REP MOVSQ) based variant.

 *

 * Calls to this get patched into the kernel image via the

 * This gets patched over the unrolled variant (below) via the

 * alternative instructions framework:

 */

	ALIGN

memcpy_c:

	CFI_STARTPROC

	.section .altinstr_replacement, "ax", @progbits

.Lmemcpy_c:

	movq %rdi, %rax



	movl %edx, %ecx
@@ -35,8 +34,8 @@ memcpy_c: 
	movl %edx, %ecx

	rep movsb

	ret

	CFI_ENDPROC

ENDPROC(memcpy_c)

.Lmemcpy_e:

	.previous



ENTRY(__memcpy)

ENTRY(memcpy)
@@ -128,16 +127,10 @@ ENDPROC(__memcpy) 
	 * It is also a lot simpler. Use this when possible:

	 */



	.section .altinstr_replacement, "ax"

1:	.byte 0xeb				/* jmp <disp8> */

	.byte (memcpy_c - memcpy) - (2f - 1b)	/* offset */

2:

	.previous



	.section .altinstructions, "a"

	.align 8

	.quad memcpy

	.quad 1b

	.quad .Lmemcpy_c

	.byte X86_FEATURE_REP_GOOD



	/*
@@ -145,6 +138,6 @@ ENDPROC(__memcpy) 
	 * so it is silly to overwrite itself with nops - reboot is the

	 * only outcome...

	 */

	.byte 2b - 1b

	.byte 2b - 1b

	.byte .Lmemcpy_e - .Lmemcpy_c

	.byte .Lmemcpy_e - .Lmemcpy_c

	.previous

arch/x86/lib/memset_64.S

+6 −12
Original line number Diff line number Diff line
@@ -12,9 +12,8 @@ 
 * 

 * rax   original destination

 */	

	ALIGN

memset_c:

	CFI_STARTPROC

	.section .altinstr_replacement, "ax", @progbits

.Lmemset_c:

	movq %rdi,%r9

	movl %edx,%r8d

	andl $7,%r8d
@@ -29,8 +28,8 @@ memset_c: 
	rep stosb

	movq %r9,%rax

	ret

	CFI_ENDPROC

ENDPROC(memset_c)

.Lmemset_e:

	.previous



ENTRY(memset)

ENTRY(__memset)
@@ -118,16 +117,11 @@ ENDPROC(__memset) 


#include <asm/cpufeature.h>



	.section .altinstr_replacement,"ax"

1:	.byte 0xeb				/* jmp <disp8> */

	.byte (memset_c - memset) - (2f - 1b)	/* offset */

2:

	.previous

	.section .altinstructions,"a"

	.align 8

	.quad memset

	.quad 1b

	.quad .Lmemset_c

	.byte X86_FEATURE_REP_GOOD

	.byte .Lfinal - memset

	.byte 2b - 1b

	.byte .Lmemset_e - .Lmemset_c

	.previous

CRA Git | Maintained and supported by SUSTech CRA and CCSE