x86, mem: copy_user_64.S: Support copy_to/from_user by enhanced REP MOVSB/STOSB (4307bec9) · Commits · 戴 / test

arch/x86/lib/copy_user_64.S

+55 −10

Original line number	Original line	Diff line number	Diff line
	@@ -15,23 +15,30 @@
	#include <asm/asm-offsets.h>		#include <asm/asm-offsets.h>
	#include <asm/thread_info.h>		#include <asm/thread_info.h>
	#include <asm/cpufeature.h>		#include <asm/cpufeature.h>
			#include <asm/alternative-asm.h>

	.macro ALTERNATIVE_JUMP feature,orig,alt		/*
			* By placing feature2 after feature1 in altinstructions section, we logically
			* implement:
			* If CPU has feature2, jmp to alt2 is used
			* else if CPU has feature1, jmp to alt1 is used
			* else jmp to orig is used.
			*/
			.macro ALTERNATIVE_JUMP feature1,feature2,orig,alt1,alt2
	0:		0:
	.byte 0xe9 /* 32bit jump */		.byte 0xe9 /* 32bit jump */
	.long \orig-1f /* by default jump to orig */		.long \orig-1f /* by default jump to orig */
	1:		1:
	.section .altinstr_replacement,"ax"		.section .altinstr_replacement,"ax"
	2: .byte 0xe9 /* near jump with 32bit immediate */		2: .byte 0xe9 /* near jump with 32bit immediate */
	.long \alt-1b /* offset / / or alternatively to alt */		.long \alt1-1b /* offset / / or alternatively to alt1 */
			3: .byte 0xe9 /* near jump with 32bit immediate */
			.long \alt2-1b /* offset / / or alternatively to alt2 */
	.previous		.previous

	.section .altinstructions,"a"		.section .altinstructions,"a"
	.align 8		altinstruction_entry 0b,2b,\feature1,5,5
	.quad 0b		altinstruction_entry 0b,3b,\feature2,5,5
	.quad 2b
	.word \feature /* when feature is set */
	.byte 5
	.byte 5
	.previous		.previous
	.endm		.endm

	@@ -73,7 +80,9 @@ ENTRY(_copy_to_user)
	jc bad_to_user		jc bad_to_user
	cmpq TI_addr_limit(%rax),%rcx		cmpq TI_addr_limit(%rax),%rcx
	jae bad_to_user		jae bad_to_user
	ALTERNATIVE_JUMP X86_FEATURE_REP_GOOD,copy_user_generic_unrolled,copy_user_generic_string		ALTERNATIVE_JUMP X86_FEATURE_REP_GOOD,X86_FEATURE_ERMS, \
			copy_user_generic_unrolled,copy_user_generic_string, \
			copy_user_enhanced_fast_string
	CFI_ENDPROC		CFI_ENDPROC
	ENDPROC(_copy_to_user)		ENDPROC(_copy_to_user)

	@@ -86,7 +95,9 @@ ENTRY(_copy_from_user)
	jc bad_from_user		jc bad_from_user
	cmpq TI_addr_limit(%rax),%rcx		cmpq TI_addr_limit(%rax),%rcx
	jae bad_from_user		jae bad_from_user
	ALTERNATIVE_JUMP X86_FEATURE_REP_GOOD,copy_user_generic_unrolled,copy_user_generic_string		ALTERNATIVE_JUMP X86_FEATURE_REP_GOOD,X86_FEATURE_ERMS, \
			copy_user_generic_unrolled,copy_user_generic_string, \
			copy_user_enhanced_fast_string
	CFI_ENDPROC		CFI_ENDPROC
	ENDPROC(_copy_from_user)		ENDPROC(_copy_from_user)

	@@ -255,3 +266,37 @@ ENTRY(copy_user_generic_string)
	.previous		.previous
	CFI_ENDPROC		CFI_ENDPROC
	ENDPROC(copy_user_generic_string)		ENDPROC(copy_user_generic_string)

			/*
			* Some CPUs are adding enhanced REP MOVSB/STOSB instructions.
			* It's recommended to use enhanced REP MOVSB/STOSB if it's enabled.
			*
			* Input:
			* rdi destination
			* rsi source
			* rdx count
			*
			* Output:
			* eax uncopied bytes or 0 if successful.
			*/
			ENTRY(copy_user_enhanced_fast_string)
			CFI_STARTPROC
			andl %edx,%edx
			jz 2f
			movl %edx,%ecx
			1: rep
			movsb
			2: xorl %eax,%eax
			ret

			.section .fixup,"ax"
			12: movl %ecx,%edx /* ecx is zerorest also */
			jmp copy_user_handle_tail
			.previous

			.section __ex_table,"a"
			.align 8
			.quad 1b,12b
			.previous
			CFI_ENDPROC
			ENDPROC(copy_user_enhanced_fast_string)

Admin message