Commit 4b842e4e authored by Al Viro's avatar Al Viro
Browse files

x86: get rid of small constant size cases in raw_copy_{to,from}_user() 



Very few call sites where that would be triggered remain, and none
of those is anywhere near hot enough to bother.

Signed-off-by: default avatarAl Viro <viro@zeniv.linux.org.uk>
parent 71c3313a

arch/x86/include/asm/uaccess.h

+0 −12
Original line number Diff line number Diff line
@@ -378,18 +378,6 @@ do { \ 
		     : "=r" (err), ltype(x)				\

		     : "m" (__m(addr)), "i" (errret), "0" (err))



#define __get_user_asm_nozero(x, addr, err, itype, rtype, ltype, errret)	\

	asm volatile("\n"						\

		     "1:	mov"itype" %2,%"rtype"1\n"		\

		     "2:\n"						\

		     ".section .fixup,\"ax\"\n"				\

		     "3:	mov %3,%0\n"				\

		     "	jmp 2b\n"					\

		     ".previous\n"					\

		     _ASM_EXTABLE_UA(1b, 3b)				\

		     : "=r" (err), ltype(x)				\

		     : "m" (__m(addr)), "i" (errret), "0" (err))



/*

 * This doesn't do __uaccess_begin/end - the exception handling

 * around it must do that.

arch/x86/include/asm/uaccess_32.h

+0 −27
Original line number Diff line number Diff line
@@ -23,33 +23,6 @@ raw_copy_to_user(void __user *to, const void *from, unsigned long n) 
static __always_inline unsigned long

raw_copy_from_user(void *to, const void __user *from, unsigned long n)

{

	if (__builtin_constant_p(n)) {

		unsigned long ret;



		switch (n) {

		case 1:

			ret = 0;

			__uaccess_begin_nospec();

			__get_user_asm_nozero(*(u8 *)to, from, ret,

					      "b", "b", "=q", 1);

			__uaccess_end();

			return ret;

		case 2:

			ret = 0;

			__uaccess_begin_nospec();

			__get_user_asm_nozero(*(u16 *)to, from, ret,

					      "w", "w", "=r", 2);

			__uaccess_end();

			return ret;

		case 4:

			ret = 0;

			__uaccess_begin_nospec();

			__get_user_asm_nozero(*(u32 *)to, from, ret,

					      "l", "k", "=r", 4);

			__uaccess_end();

			return ret;

		}

	}

	return __copy_user_ll(to, (__force const void *)from, n);

}

arch/x86/include/asm/uaccess_64.h

+2 −106
Original line number Diff line number Diff line
@@ -65,118 +65,14 @@ copy_to_user_mcsafe(void *to, const void *from, unsigned len) 
static __always_inline __must_check unsigned long

raw_copy_from_user(void *dst, const void __user *src, unsigned long size)

{

	int ret = 0;



	if (!__builtin_constant_p(size))

	return copy_user_generic(dst, (__force void *)src, size);

	switch (size) {

	case 1:

		__uaccess_begin_nospec();

		__get_user_asm_nozero(*(u8 *)dst, (u8 __user *)src,

			      ret, "b", "b", "=q", 1);

		__uaccess_end();

		return ret;

	case 2:

		__uaccess_begin_nospec();

		__get_user_asm_nozero(*(u16 *)dst, (u16 __user *)src,

			      ret, "w", "w", "=r", 2);

		__uaccess_end();

		return ret;

	case 4:

		__uaccess_begin_nospec();

		__get_user_asm_nozero(*(u32 *)dst, (u32 __user *)src,

			      ret, "l", "k", "=r", 4);

		__uaccess_end();

		return ret;

	case 8:

		__uaccess_begin_nospec();

		__get_user_asm_nozero(*(u64 *)dst, (u64 __user *)src,

			      ret, "q", "", "=r", 8);

		__uaccess_end();

		return ret;

	case 10:

		__uaccess_begin_nospec();

		__get_user_asm_nozero(*(u64 *)dst, (u64 __user *)src,

			       ret, "q", "", "=r", 10);

		if (likely(!ret))

			__get_user_asm_nozero(*(u16 *)(8 + (char *)dst),

				       (u16 __user *)(8 + (char __user *)src),

				       ret, "w", "w", "=r", 2);

		__uaccess_end();

		return ret;

	case 16:

		__uaccess_begin_nospec();

		__get_user_asm_nozero(*(u64 *)dst, (u64 __user *)src,

			       ret, "q", "", "=r", 16);

		if (likely(!ret))

			__get_user_asm_nozero(*(u64 *)(8 + (char *)dst),

				       (u64 __user *)(8 + (char __user *)src),

				       ret, "q", "", "=r", 8);

		__uaccess_end();

		return ret;

	default:

		return copy_user_generic(dst, (__force void *)src, size);

	}

}



static __always_inline __must_check unsigned long

raw_copy_to_user(void __user *dst, const void *src, unsigned long size)

{

	int ret = 0;



	if (!__builtin_constant_p(size))

		return copy_user_generic((__force void *)dst, src, size);

	switch (size) {

	case 1:

		__uaccess_begin();

		__put_user_asm(*(u8 *)src, (u8 __user *)dst,

			      ret, "b", "b", "iq", 1);

		__uaccess_end();

		return ret;

	case 2:

		__uaccess_begin();

		__put_user_asm(*(u16 *)src, (u16 __user *)dst,

			      ret, "w", "w", "ir", 2);

		__uaccess_end();

		return ret;

	case 4:

		__uaccess_begin();

		__put_user_asm(*(u32 *)src, (u32 __user *)dst,

			      ret, "l", "k", "ir", 4);

		__uaccess_end();

		return ret;

	case 8:

		__uaccess_begin();

		__put_user_asm(*(u64 *)src, (u64 __user *)dst,

			      ret, "q", "", "er", 8);

		__uaccess_end();

		return ret;

	case 10:

		__uaccess_begin();

		__put_user_asm(*(u64 *)src, (u64 __user *)dst,

			       ret, "q", "", "er", 10);

		if (likely(!ret)) {

			asm("":::"memory");

			__put_user_asm(4[(u16 *)src], 4 + (u16 __user *)dst,

				       ret, "w", "w", "ir", 2);

		}

		__uaccess_end();

		return ret;

	case 16:

		__uaccess_begin();

		__put_user_asm(*(u64 *)src, (u64 __user *)dst,

			       ret, "q", "", "er", 16);

		if (likely(!ret)) {

			asm("":::"memory");

			__put_user_asm(1[(u64 *)src], 1 + (u64 __user *)dst,

				       ret, "q", "", "er", 8);

		}

		__uaccess_end();

		return ret;

	default:

	return copy_user_generic((__force void *)dst, src, size);

}

}



static __always_inline __must_check

unsigned long raw_copy_in_user(void __user *dst, const void __user *src, unsigned long size)

CRA Git | Maintained and supported by SUSTech CRA and CCSE