Commit ea6f043f authored by Linus Torvalds's avatar Linus Torvalds
Browse files

x86: Make __get_user() generate an out-of-line call 



Instead of inlining the whole stac/lfence/mov/clac sequence (which also
requires individual exception table entries and several asm instruction
alternatives entries), just generate "call __get_user_nocheck_X" for the
__get_user() cases.

We can use all the same infrastructure that we already do for the
regular "get_user()", and the end result is simpler source code, and
much simpler code generation.

It also means that when I introduce asm goto with input for
"unsafe_get_user()", there are no nasty interactions with the
__get_user() code.

Cc: Al Viro <viro@zeniv.linux.org.uk>
Cc: Thomas Gleixner <tglx@linutronix.de>
Cc: Ingo Molnar <mingo@kernel.org>
Cc: Borislav Petkov <bp@alien8.de>
Cc: Peter Zijlstra <peterz@infradead.org>
Signed-off-by: default avatarLinus Torvalds <torvalds@linux-foundation.org>
parent 53acd350

arch/x86/include/asm/uaccess.h

+54 −74
Original line number Diff line number Diff line
@@ -96,25 +96,14 @@ static inline bool pagefault_disabled(void); 
	likely(!__range_not_ok(addr, size, user_addr_max()));		\

})



/*

 * These are the main single-value transfer routines.  They automatically

 * use the right size if we just have the right pointer type.

 *

 * This gets kind of ugly. We want to return _two_ values in "get_user()"

 * and yet we don't want to do any pointers, because that is too much

 * of a performance impact. Thus we have a few rather ugly macros here,

 * and hide all the ugliness from the user.

 *

 * The "__xxx" versions of the user access functions are versions that

 * do not verify the address space, that must have been done previously

 * with a separate "access_ok()" call (this is used when we do multiple

 * accesses to the same area of user memory).

 */



extern int __get_user_1(void);

extern int __get_user_2(void);

extern int __get_user_4(void);

extern int __get_user_8(void);

extern int __get_user_nocheck_1(void);

extern int __get_user_nocheck_2(void);

extern int __get_user_nocheck_4(void);

extern int __get_user_nocheck_8(void);

extern int __get_user_bad(void);



#define __uaccess_begin() stac()
@@ -138,25 +127,12 @@ extern int __get_user_bad(void); 
#define __typefits(x,type,not) \

	__builtin_choose_expr(sizeof(x)<=sizeof(type),(unsigned type)0,not)



/**

 * get_user - Get a simple variable from user space.

 * @x:   Variable to store result.

 * @ptr: Source address, in user space.

 *

 * Context: User context only. This function may sleep if pagefaults are

 *          enabled.

 *

 * This macro copies a single simple variable from user space to kernel

 * space.  It supports simple types like char and int, but not larger

 * data types like structures or arrays.

 *

 * @ptr must have pointer-to-simple-variable type, and the result of

 * dereferencing @ptr must be assignable to @x without a cast.

 *

 * Return: zero on success, or -EFAULT on error.

 * On error, the variable @x is set to zero.

 */

/*

 * This is used for both get_user() and __get_user() to expand to

 * the proper special function call that has odd calling conventions

 * due to returning both a value and an error, and that depends on

 * the size of the pointer passed in.

 *

 * Careful: we have to cast the result to the type of the pointer

 * for sign reasons.

 *
@@ -169,13 +145,12 @@ extern int __get_user_bad(void); 
 * Clang/LLVM cares about the size of the register, but still wants

 * the base register for something that ends up being a pair.

 */

#define get_user(x, ptr)						\

#define do_get_user_call(fn,x,ptr)					\

({									\

	int __ret_gu;							\

	register __inttype(*(ptr)) __val_gu asm("%"_ASM_DX);		\

	__chk_user_ptr(ptr);						\

	might_fault();							\

	asm volatile("call __get_user_%P4"				\

	asm volatile("call __" #fn "_%P4"				\

		     : "=a" (__ret_gu), "=r" (__val_gu),		\

			ASM_CALL_CONSTRAINT				\

		     : "0" (ptr), "i" (sizeof(*(ptr))));		\
@@ -183,6 +158,49 @@ extern int __get_user_bad(void); 
	__builtin_expect(__ret_gu, 0);					\

})



/**

 * get_user - Get a simple variable from user space.

 * @x:   Variable to store result.

 * @ptr: Source address, in user space.

 *

 * Context: User context only. This function may sleep if pagefaults are

 *          enabled.

 *

 * This macro copies a single simple variable from user space to kernel

 * space.  It supports simple types like char and int, but not larger

 * data types like structures or arrays.

 *

 * @ptr must have pointer-to-simple-variable type, and the result of

 * dereferencing @ptr must be assignable to @x without a cast.

 *

 * Return: zero on success, or -EFAULT on error.

 * On error, the variable @x is set to zero.

 */

#define get_user(x,ptr) ({ might_fault(); do_get_user_call(get_user,x,ptr); })



/**

 * __get_user - Get a simple variable from user space, with less checking.

 * @x:   Variable to store result.

 * @ptr: Source address, in user space.

 *

 * Context: User context only. This function may sleep if pagefaults are

 *          enabled.

 *

 * This macro copies a single simple variable from user space to kernel

 * space.  It supports simple types like char and int, but not larger

 * data types like structures or arrays.

 *

 * @ptr must have pointer-to-simple-variable type, and the result of

 * dereferencing @ptr must be assignable to @x without a cast.

 *

 * Caller must check the pointer with access_ok() before calling this

 * function.

 *

 * Return: zero on success, or -EFAULT on error.

 * On error, the variable @x is set to zero.

 */

#define __get_user(x,ptr) do_get_user_call(get_user_nocheck,x,ptr)



#define __put_user_x(size, x, ptr, __ret_pu)			\

	asm volatile("call __put_user_" #size : "=a" (__ret_pu)	\

		     : "0" ((typeof(*(ptr)))(x)), "c" (ptr) : "ebx")
@@ -367,19 +385,6 @@ __pu_label: \ 
	__builtin_expect(__pu_err, 0);				\

})



#define __get_user_nocheck(x, ptr, size)				\

({									\

	int __gu_err;							\

	__inttype(*(ptr)) __gu_val;					\

	__typeof__(ptr) __gu_ptr = (ptr);				\

	__typeof__(size) __gu_size = (size);				\

	__uaccess_begin_nospec();					\

	__get_user_size(__gu_val, __gu_ptr, __gu_size, __gu_err);	\

	__uaccess_end();						\

	(x) = (__force __typeof__(*(ptr)))__gu_val;			\

	__builtin_expect(__gu_err, 0);					\

})



/* FIXME: this hack is definitely wrong -AK */

struct __large_struct { unsigned long buf[100]; };

#define __m(x) (*(struct __large_struct __user *)(x))
@@ -396,31 +401,6 @@ struct __large_struct { unsigned long buf[100]; }; 
		: : ltype(x), "m" (__m(addr))				\

		: : label)



/**

 * __get_user - Get a simple variable from user space, with less checking.

 * @x:   Variable to store result.

 * @ptr: Source address, in user space.

 *

 * Context: User context only. This function may sleep if pagefaults are

 *          enabled.

 *

 * This macro copies a single simple variable from user space to kernel

 * space.  It supports simple types like char and int, but not larger

 * data types like structures or arrays.

 *

 * @ptr must have pointer-to-simple-variable type, and the result of

 * dereferencing @ptr must be assignable to @x without a cast.

 *

 * Caller must check the pointer with access_ok() before calling this

 * function.

 *

 * Return: zero on success, or -EFAULT on error.

 * On error, the variable @x is set to zero.

 */



#define __get_user(x, ptr)						\

	__get_user_nocheck((x), (ptr), sizeof(*(ptr)))



/**

 * __put_user - Write a simple value into user space, with less checking.

 * @x:   Value to copy to user space.

arch/x86/lib/getuser.S

+60 −0
Original line number Diff line number Diff line
@@ -35,6 +35,8 @@ 
#include <asm/smap.h>

#include <asm/export.h>



#define ASM_BARRIER_NOSPEC ALTERNATIVE "", "lfence", X86_FEATURE_LFENCE_RDTSC



	.text

SYM_FUNC_START(__get_user_1)

	mov PER_CPU_VAR(current_task), %_ASM_DX
@@ -114,6 +116,52 @@ SYM_FUNC_START(__get_user_8) 
SYM_FUNC_END(__get_user_8)

EXPORT_SYMBOL(__get_user_8)



/* .. and the same for __get_user, just without the range checks */

SYM_FUNC_START(__get_user_nocheck_1)

	ASM_STAC

	ASM_BARRIER_NOSPEC

6:	movzbl (%_ASM_AX),%edx

	xor %eax,%eax

	ASM_CLAC

	ret

SYM_FUNC_END(__get_user_nocheck_1)

EXPORT_SYMBOL(__get_user_nocheck_1)



SYM_FUNC_START(__get_user_nocheck_2)

	ASM_STAC

	ASM_BARRIER_NOSPEC

7:	movzwl (%_ASM_AX),%edx

	xor %eax,%eax

	ASM_CLAC

	ret

SYM_FUNC_END(__get_user_nocheck_2)

EXPORT_SYMBOL(__get_user_nocheck_2)



SYM_FUNC_START(__get_user_nocheck_4)

	ASM_STAC

	ASM_BARRIER_NOSPEC

8:	movl (%_ASM_AX),%edx

	xor %eax,%eax

	ASM_CLAC

	ret

SYM_FUNC_END(__get_user_nocheck_4)

EXPORT_SYMBOL(__get_user_nocheck_4)



SYM_FUNC_START(__get_user_nocheck_8)

	ASM_STAC

	ASM_BARRIER_NOSPEC

#ifdef CONFIG_X86_64

9:	movq (%_ASM_AX),%rdx

#else

9:	movl (%_ASM_AX),%edx

10:	movl 4(%_ASM_AX),%ecx

#endif

	xor %eax,%eax

	ASM_CLAC

	ret

SYM_FUNC_END(__get_user_nocheck_8)

EXPORT_SYMBOL(__get_user_nocheck_8)





SYM_CODE_START_LOCAL(.Lbad_get_user_clac)

	ASM_CLAC
@@ -134,6 +182,7 @@ bad_get_user_8: 
SYM_CODE_END(.Lbad_get_user_8_clac)

#endif



/* get_user */

	_ASM_EXTABLE_UA(1b, .Lbad_get_user_clac)

	_ASM_EXTABLE_UA(2b, .Lbad_get_user_clac)

	_ASM_EXTABLE_UA(3b, .Lbad_get_user_clac)
@@ -143,3 +192,14 @@ SYM_CODE_END(.Lbad_get_user_8_clac) 
	_ASM_EXTABLE_UA(4b, .Lbad_get_user_8_clac)

	_ASM_EXTABLE_UA(5b, .Lbad_get_user_8_clac)

#endif



/* __get_user */

	_ASM_EXTABLE_UA(6b, .Lbad_get_user_clac)

	_ASM_EXTABLE_UA(7b, .Lbad_get_user_clac)

	_ASM_EXTABLE_UA(8b, .Lbad_get_user_clac)

#ifdef CONFIG_X86_64

	_ASM_EXTABLE_UA(9b, .Lbad_get_user_clac)

#else

	_ASM_EXTABLE_UA(9b, .Lbad_get_user_8_clac)

	_ASM_EXTABLE_UA(10b, .Lbad_get_user_8_clac)

#endif

CRA Git | Maintained and supported by SUSTech CRA and CCSE