Commit d55564cf authored by Linus Torvalds's avatar Linus Torvalds
Browse files

x86: Make __put_user() generate an out-of-line call 



Instead of inlining the stac/mov/clac sequence (which also requires
individual exception table entries and several asm instruction
alternatives entries), just generate "call __put_user_nocheck_X" for the
__put_user() cases, the same way we changed __get_user earlier.

Unlike the get_user() case, we didn't have the same nice infrastructure
to just generate the call with a single case, so this actually has to
change some of the infrastructure in order to do this.  But that only
cleans up the code further.

So now, instead of using a case statement for the sizes, we just do the
same thing we've done on the get_user() side for a long time: use the
size as an immediate constant to the asm, and generate the asm that way
directly.

In order to handle the special case of 64-bit data on a 32-bit kernel, I
needed to change the calling convention slightly: the data is passed in
%eax[:%edx], the pointer in %ecx, and the return value is also returned
in %ecx.  It used to be returned in %eax, but because of how %eax can
now be a double register input, we don't want mix that with a
single-register output.

The actual low-level asm is easier to handle: we'll just share the code
between the checking and non-checking case, with the non-checking case
jumping into the middle of the function.  That may sound a bit too
special, but this code is all very very special anyway, so...

Cc: Al Viro <viro@zeniv.linux.org.uk>
Cc: Thomas Gleixner <tglx@linutronix.de>
Cc: Ingo Molnar <mingo@kernel.org>
Cc: Borislav Petkov <bp@alien8.de>
Cc: Peter Zijlstra <peterz@infradead.org>
Signed-off-by: default avatarLinus Torvalds <torvalds@linux-foundation.org>
parent ea6f043f

arch/x86/include/asm/uaccess.h

+44 −75
Original line number Diff line number Diff line
@@ -201,11 +201,6 @@ extern int __get_user_bad(void); 
 */

#define __get_user(x,ptr) do_get_user_call(get_user_nocheck,x,ptr)



#define __put_user_x(size, x, ptr, __ret_pu)			\

	asm volatile("call __put_user_" #size : "=a" (__ret_pu)	\

		     : "0" ((typeof(*(ptr)))(x)), "c" (ptr) : "ebx")







#ifdef CONFIG_X86_32

#define __put_user_goto_u64(x, addr, label)			\
@@ -217,25 +212,41 @@ extern int __get_user_bad(void); 
		     : : "A" (x), "r" (addr)			\

		     : : label)



#define __put_user_x8(x, ptr, __ret_pu)				\

	asm volatile("call __put_user_8" : "=a" (__ret_pu)	\

		     : "A" ((typeof(*(ptr)))(x)), "c" (ptr) : "ebx")

#else

#define __put_user_goto_u64(x, ptr, label) \

	__put_user_goto(x, ptr, "q", "er", label)

#define __put_user_x8(x, ptr, __ret_pu) __put_user_x(8, x, ptr, __ret_pu)

#endif



extern void __put_user_bad(void);



/*

 * Strange magic calling convention: pointer in %ecx,

 * value in %eax(:%edx), return value in %eax. clobbers %rbx

 * value in %eax(:%edx), return value in %ecx. clobbers %rbx

 */

extern void __put_user_1(void);

extern void __put_user_2(void);

extern void __put_user_4(void);

extern void __put_user_8(void);

extern void __put_user_nocheck_1(void);

extern void __put_user_nocheck_2(void);

extern void __put_user_nocheck_4(void);

extern void __put_user_nocheck_8(void);



#define do_put_user_call(fn,x,ptr)					\

({									\

	int __ret_pu;							\

	register __typeof__(*(ptr)) __val_pu asm("%"_ASM_AX);		\

	__chk_user_ptr(ptr);						\

	__val_pu = (x);							\

	asm volatile("call __" #fn "_%P[size]"				\

		     : "=c" (__ret_pu),					\

			ASM_CALL_CONSTRAINT				\

		     : "0" (ptr),					\

		       "r" (__val_pu),					\

		       [size] "i" (sizeof(*(ptr)))			\

		     :"ebx");						\

	__builtin_expect(__ret_pu, 0);					\

})



/**

 * put_user - Write a simple value into user space.
@@ -254,32 +265,29 @@ extern void __put_user_8(void); 
 *

 * Return: zero on success, or -EFAULT on error.

 */

#define put_user(x, ptr)					\

({								\

	int __ret_pu;						\

	__typeof__(*(ptr)) __pu_val;				\

	__chk_user_ptr(ptr);					\

	might_fault();						\

	__pu_val = x;						\

	switch (sizeof(*(ptr))) {				\

	case 1:							\

		__put_user_x(1, __pu_val, ptr, __ret_pu);	\

		break;						\

	case 2:							\

		__put_user_x(2, __pu_val, ptr, __ret_pu);	\

		break;						\

	case 4:							\

		__put_user_x(4, __pu_val, ptr, __ret_pu);	\

		break;						\

	case 8:							\

		__put_user_x8(__pu_val, ptr, __ret_pu);		\

		break;						\

	default:						\

		__put_user_x(X, __pu_val, ptr, __ret_pu);	\

		break;						\

	}							\

	__builtin_expect(__ret_pu, 0);				\

})

#define put_user(x, ptr) ({ might_fault(); do_put_user_call(put_user,x,ptr); })



/**

 * __put_user - Write a simple value into user space, with less checking.

 * @x:   Value to copy to user space.

 * @ptr: Destination address, in user space.

 *

 * Context: User context only. This function may sleep if pagefaults are

 *          enabled.

 *

 * This macro copies a single simple value from kernel space to user

 * space.  It supports simple types like char and int, but not larger

 * data types like structures or arrays.

 *

 * @ptr must have pointer-to-simple-variable type, and @x must be assignable

 * to the result of dereferencing @ptr.

 *

 * Caller must check the pointer with access_ok() before calling this

 * function.

 *

 * Return: zero on success, or -EFAULT on error.

 */

#define __put_user(x, ptr) do_put_user_call(put_user_nocheck,x,ptr)



#define __put_user_size(x, ptr, size, label)				\

do {									\
@@ -370,21 +378,6 @@ do { \ 
		     : [umem] "m" (__m(addr)),				\

		       [efault] "i" (-EFAULT), "0" (err))



#define __put_user_nocheck(x, ptr, size)			\

({								\

	__label__ __pu_label;					\

	int __pu_err = -EFAULT;					\

	__typeof__(*(ptr)) __pu_val = (x);			\

	__typeof__(ptr) __pu_ptr = (ptr);			\

	__typeof__(size) __pu_size = (size);			\

	__uaccess_begin();					\

	__put_user_size(__pu_val, __pu_ptr, __pu_size, __pu_label);	\

	__pu_err = 0;						\

__pu_label:							\

	__uaccess_end();					\

	__builtin_expect(__pu_err, 0);				\

})



/* FIXME: this hack is definitely wrong -AK */

struct __large_struct { unsigned long buf[100]; };

#define __m(x) (*(struct __large_struct __user *)(x))
@@ -401,30 +394,6 @@ struct __large_struct { unsigned long buf[100]; }; 
		: : ltype(x), "m" (__m(addr))				\

		: : label)



/**

 * __put_user - Write a simple value into user space, with less checking.

 * @x:   Value to copy to user space.

 * @ptr: Destination address, in user space.

 *

 * Context: User context only. This function may sleep if pagefaults are

 *          enabled.

 *

 * This macro copies a single simple value from kernel space to user

 * space.  It supports simple types like char and int, but not larger

 * data types like structures or arrays.

 *

 * @ptr must have pointer-to-simple-variable type, and @x must be assignable

 * to the result of dereferencing @ptr.

 *

 * Caller must check the pointer with access_ok() before calling this

 * function.

 *

 * Return: zero on success, or -EFAULT on error.

 */



#define __put_user(x, ptr)						\

	__put_user_nocheck((__typeof__(*(ptr)))(x), (ptr), sizeof(*(ptr)))



extern unsigned long

copy_from_user_nmi(void *to, const void __user *from, unsigned long n);

extern __must_check long

arch/x86/lib/putuser.S

+16 −6
Original line number Diff line number Diff line
@@ -25,7 +25,9 @@ 
 * Inputs:	%eax[:%edx] contains the data

 *		%ecx contains the address

 *

 * Outputs:	%eax is error code (0 or -EFAULT)

 * Outputs:	%ecx is error code (0 or -EFAULT)

 *

 * Clobbers:	%ebx needed for task pointer

 *

 * These functions should not modify any other registers,

 * as they get called from within inline assembly.
@@ -38,13 +40,15 @@ SYM_FUNC_START(__put_user_1) 
	ENTER

	cmp TASK_addr_limit(%_ASM_BX),%_ASM_CX

	jae .Lbad_put_user

SYM_INNER_LABEL(__put_user_nocheck_1, SYM_L_GLOBAL)

	ASM_STAC

1:	movb %al,(%_ASM_CX)

	xor %eax,%eax

	xor %ecx,%ecx

	ASM_CLAC

	ret

SYM_FUNC_END(__put_user_1)

EXPORT_SYMBOL(__put_user_1)

EXPORT_SYMBOL(__put_user_nocheck_1)



SYM_FUNC_START(__put_user_2)

	ENTER
@@ -52,13 +56,15 @@ SYM_FUNC_START(__put_user_2) 
	sub $1,%_ASM_BX

	cmp %_ASM_BX,%_ASM_CX

	jae .Lbad_put_user

SYM_INNER_LABEL(__put_user_nocheck_2, SYM_L_GLOBAL)

	ASM_STAC

2:	movw %ax,(%_ASM_CX)

	xor %eax,%eax

	xor %ecx,%ecx

	ASM_CLAC

	ret

SYM_FUNC_END(__put_user_2)

EXPORT_SYMBOL(__put_user_2)

EXPORT_SYMBOL(__put_user_nocheck_2)



SYM_FUNC_START(__put_user_4)

	ENTER
@@ -66,13 +72,15 @@ SYM_FUNC_START(__put_user_4) 
	sub $3,%_ASM_BX

	cmp %_ASM_BX,%_ASM_CX

	jae .Lbad_put_user

SYM_INNER_LABEL(__put_user_nocheck_4, SYM_L_GLOBAL)

	ASM_STAC

3:	movl %eax,(%_ASM_CX)

	xor %eax,%eax

	xor %ecx,%ecx

	ASM_CLAC

	ret

SYM_FUNC_END(__put_user_4)

EXPORT_SYMBOL(__put_user_4)

EXPORT_SYMBOL(__put_user_nocheck_4)



SYM_FUNC_START(__put_user_8)

	ENTER
@@ -80,21 +88,23 @@ SYM_FUNC_START(__put_user_8) 
	sub $7,%_ASM_BX

	cmp %_ASM_BX,%_ASM_CX

	jae .Lbad_put_user

SYM_INNER_LABEL(__put_user_nocheck_8, SYM_L_GLOBAL)

	ASM_STAC

4:	mov %_ASM_AX,(%_ASM_CX)

#ifdef CONFIG_X86_32

5:	movl %edx,4(%_ASM_CX)

#endif

	xor %eax,%eax

	xor %ecx,%ecx

	ASM_CLAC

	RET

SYM_FUNC_END(__put_user_8)

EXPORT_SYMBOL(__put_user_8)

EXPORT_SYMBOL(__put_user_nocheck_8)



SYM_CODE_START_LOCAL(.Lbad_put_user_clac)

	ASM_CLAC

.Lbad_put_user:

	movl $-EFAULT,%eax

	movl $-EFAULT,%ecx

	RET

SYM_CODE_END(.Lbad_put_user_clac)

CRA Git | Maintained and supported by SUSTech CRA and CCSE