Commit 11f1a4b9 authored by Linus Torvalds's avatar Linus Torvalds
Browse files

x86: reorganize SMAP handling in user space accesses 



This reorganizes how we do the stac/clac instructions in the user access
code.  Instead of adding the instructions directly to the same inline
asm that does the actual user level access and exception handling, add
them at a higher level.

This is mainly preparation for the next step, where we will expose an
interface to allow users to mark several accesses together as being user
space accesses, but it does already clean up some code:

 - the inlined trivial cases of copy_in_user() now do stac/clac just
   once over the accesses: they used to do one pair around the user
   space read, and another pair around the write-back.

 - the {get,put}_user_ex() macros that are used with the catch/try
   handling don't do any stac/clac at all, because that happens in the
   try/catch surrounding them.

Other than those two cleanups that happened naturally from the
re-organization, this should not make any difference. Yet.

Signed-off-by: default avatarLinus Torvalds <torvalds@linux-foundation.org>
parent a5e90b1b

arch/x86/include/asm/uaccess.h

+35 −18
Original line number Diff line number Diff line
@@ -134,6 +134,9 @@ extern int __get_user_4(void); 
extern int __get_user_8(void);

extern int __get_user_bad(void);



#define __uaccess_begin() stac()

#define __uaccess_end()   clac()



/*

 * This is a type: either unsigned long, if the argument fits into

 * that type, or otherwise unsigned long long.
@@ -193,10 +196,10 @@ __typeof__(__builtin_choose_expr(sizeof(x) > sizeof(0UL), 0ULL, 0UL)) 


#ifdef CONFIG_X86_32

#define __put_user_asm_u64(x, addr, err, errret)			\

	asm volatile(ASM_STAC "\n"					\

	asm volatile("\n"						\

		     "1:	movl %%eax,0(%2)\n"			\

		     "2:	movl %%edx,4(%2)\n"			\

		     "3: " ASM_CLAC "\n"				\

		     "3:"						\

		     ".section .fixup,\"ax\"\n"				\

		     "4:	movl %3,%0\n"				\

		     "	jmp 3b\n"					\
@@ -207,10 +210,10 @@ __typeof__(__builtin_choose_expr(sizeof(x) > sizeof(0UL), 0ULL, 0UL)) 
		     : "A" (x), "r" (addr), "i" (errret), "0" (err))



#define __put_user_asm_ex_u64(x, addr)					\

	asm volatile(ASM_STAC "\n"					\

	asm volatile("\n"						\

		     "1:	movl %%eax,0(%1)\n"			\

		     "2:	movl %%edx,4(%1)\n"			\

		     "3: " ASM_CLAC "\n"				\

		     "3:"						\

		     _ASM_EXTABLE_EX(1b, 2b)				\

		     _ASM_EXTABLE_EX(2b, 3b)				\

		     : : "A" (x), "r" (addr))
@@ -304,6 +307,10 @@ do { \ 
	}								\

} while (0)



/*

 * This doesn't do __uaccess_begin/end - the exception handling

 * around it must do that.

 */

#define __put_user_size_ex(x, ptr, size)				\

do {									\

	__chk_user_ptr(ptr);						\
@@ -358,9 +365,9 @@ do { \ 
} while (0)



#define __get_user_asm(x, addr, err, itype, rtype, ltype, errret)	\

	asm volatile(ASM_STAC "\n"					\

	asm volatile("\n"						\

		     "1:	mov"itype" %2,%"rtype"1\n"		\

		     "2: " ASM_CLAC "\n"				\

		     "2:\n"						\

		     ".section .fixup,\"ax\"\n"				\

		     "3:	mov %3,%0\n"				\

		     "	xor"itype" %"rtype"1,%"rtype"1\n"		\
@@ -370,6 +377,10 @@ do { \ 
		     : "=r" (err), ltype(x)				\

		     : "m" (__m(addr)), "i" (errret), "0" (err))



/*

 * This doesn't do __uaccess_begin/end - the exception handling

 * around it must do that.

 */

#define __get_user_size_ex(x, ptr, size)				\

do {									\

	__chk_user_ptr(ptr);						\
@@ -400,7 +411,9 @@ do { \ 
#define __put_user_nocheck(x, ptr, size)			\

({								\

	int __pu_err;						\

	__uaccess_begin();					\

	__put_user_size((x), (ptr), (size), __pu_err, -EFAULT);	\

	__uaccess_end();					\

	__builtin_expect(__pu_err, 0);				\

})
@@ -408,7 +421,9 @@ do { \ 
({									\

	int __gu_err;							\

	unsigned long __gu_val;						\

	__uaccess_begin();						\

	__get_user_size(__gu_val, (ptr), (size), __gu_err, -EFAULT);	\

	__uaccess_end();						\

	(x) = (__force __typeof__(*(ptr)))__gu_val;			\

	__builtin_expect(__gu_err, 0);					\

})
@@ -423,9 +438,9 @@ struct __large_struct { unsigned long buf[100]; }; 
 * aliasing issues.

 */

#define __put_user_asm(x, addr, err, itype, rtype, ltype, errret)	\

	asm volatile(ASM_STAC "\n"					\

	asm volatile("\n"						\

		     "1:	mov"itype" %"rtype"1,%2\n"		\

		     "2: " ASM_CLAC "\n"				\

		     "2:\n"						\

		     ".section .fixup,\"ax\"\n"				\

		     "3:	mov %3,%0\n"				\

		     "	jmp 2b\n"					\
@@ -445,11 +460,11 @@ struct __large_struct { unsigned long buf[100]; }; 
 */

#define uaccess_try	do {						\

	current_thread_info()->uaccess_err = 0;				\

	stac();								\

	__uaccess_begin();						\

	barrier();



#define uaccess_catch(err)						\

	clac();								\

	__uaccess_end();						\

	(err) |= (current_thread_info()->uaccess_err ? -EFAULT : 0);	\

} while (0)
@@ -547,12 +562,13 @@ extern void __cmpxchg_wrong_size(void) 
	__typeof__(ptr) __uval = (uval);				\

	__typeof__(*(ptr)) __old = (old);				\

	__typeof__(*(ptr)) __new = (new);				\

	__uaccess_begin();						\

	switch (size) {							\

	case 1:								\

	{								\

		asm volatile("\t" ASM_STAC "\n"				\

		asm volatile("\n"					\

			"1:\t" LOCK_PREFIX "cmpxchgb %4, %2\n"		\

			"2:\t" ASM_CLAC "\n"				\

			"2:\n"						\

			"\t.section .fixup, \"ax\"\n"			\

			"3:\tmov     %3, %0\n"				\

			"\tjmp     2b\n"				\
@@ -566,9 +582,9 @@ extern void __cmpxchg_wrong_size(void) 
	}								\

	case 2:								\

	{								\

		asm volatile("\t" ASM_STAC "\n"				\

		asm volatile("\n"					\

			"1:\t" LOCK_PREFIX "cmpxchgw %4, %2\n"		\

			"2:\t" ASM_CLAC "\n"				\

			"2:\n"						\

			"\t.section .fixup, \"ax\"\n"			\

			"3:\tmov     %3, %0\n"				\

			"\tjmp     2b\n"				\
@@ -582,9 +598,9 @@ extern void __cmpxchg_wrong_size(void) 
	}								\

	case 4:								\

	{								\

		asm volatile("\t" ASM_STAC "\n"				\

		asm volatile("\n"					\

			"1:\t" LOCK_PREFIX "cmpxchgl %4, %2\n"		\

			"2:\t" ASM_CLAC "\n"				\

			"2:\n"						\

			"\t.section .fixup, \"ax\"\n"			\

			"3:\tmov     %3, %0\n"				\

			"\tjmp     2b\n"				\
@@ -601,9 +617,9 @@ extern void __cmpxchg_wrong_size(void) 
		if (!IS_ENABLED(CONFIG_X86_64))				\

			__cmpxchg_wrong_size();				\

									\

		asm volatile("\t" ASM_STAC "\n"				\

		asm volatile("\n"					\

			"1:\t" LOCK_PREFIX "cmpxchgq %4, %2\n"		\

			"2:\t" ASM_CLAC "\n"				\

			"2:\n"						\

			"\t.section .fixup, \"ax\"\n"			\

			"3:\tmov     %3, %0\n"				\

			"\tjmp     2b\n"				\
@@ -618,6 +634,7 @@ extern void __cmpxchg_wrong_size(void) 
	default:							\

		__cmpxchg_wrong_size();					\

	}								\

	__uaccess_end();						\

	*__uval = __old;						\

	__ret;								\

})

arch/x86/include/asm/uaccess_64.h

+66 −28
Original line number Diff line number Diff line
@@ -56,35 +56,49 @@ int __copy_from_user_nocheck(void *dst, const void __user *src, unsigned size) 
	if (!__builtin_constant_p(size))

		return copy_user_generic(dst, (__force void *)src, size);

	switch (size) {

	case 1:__get_user_asm(*(u8 *)dst, (u8 __user *)src,

	case 1:

		__uaccess_begin();

		__get_user_asm(*(u8 *)dst, (u8 __user *)src,

			      ret, "b", "b", "=q", 1);

		__uaccess_end();

		return ret;

	case 2:__get_user_asm(*(u16 *)dst, (u16 __user *)src,

	case 2:

		__uaccess_begin();

		__get_user_asm(*(u16 *)dst, (u16 __user *)src,

			      ret, "w", "w", "=r", 2);

		__uaccess_end();

		return ret;

	case 4:__get_user_asm(*(u32 *)dst, (u32 __user *)src,

	case 4:

		__uaccess_begin();

		__get_user_asm(*(u32 *)dst, (u32 __user *)src,

			      ret, "l", "k", "=r", 4);

		__uaccess_end();

		return ret;

	case 8:__get_user_asm(*(u64 *)dst, (u64 __user *)src,

	case 8:

		__uaccess_begin();

		__get_user_asm(*(u64 *)dst, (u64 __user *)src,

			      ret, "q", "", "=r", 8);

		__uaccess_end();

		return ret;

	case 10:

		__uaccess_begin();

		__get_user_asm(*(u64 *)dst, (u64 __user *)src,

			       ret, "q", "", "=r", 10);

		if (unlikely(ret))

			return ret;

		if (likely(!ret))

			__get_user_asm(*(u16 *)(8 + (char *)dst),

				       (u16 __user *)(8 + (char __user *)src),

				       ret, "w", "w", "=r", 2);

		__uaccess_end();

		return ret;

	case 16:

		__uaccess_begin();

		__get_user_asm(*(u64 *)dst, (u64 __user *)src,

			       ret, "q", "", "=r", 16);

		if (unlikely(ret))

			return ret;

		if (likely(!ret))

			__get_user_asm(*(u64 *)(8 + (char *)dst),

				       (u64 __user *)(8 + (char __user *)src),

				       ret, "q", "", "=r", 8);

		__uaccess_end();

		return ret;

	default:

		return copy_user_generic(dst, (__force void *)src, size);
@@ -106,35 +120,51 @@ int __copy_to_user_nocheck(void __user *dst, const void *src, unsigned size) 
	if (!__builtin_constant_p(size))

		return copy_user_generic((__force void *)dst, src, size);

	switch (size) {

	case 1:__put_user_asm(*(u8 *)src, (u8 __user *)dst,

	case 1:

		__uaccess_begin();

		__put_user_asm(*(u8 *)src, (u8 __user *)dst,

			      ret, "b", "b", "iq", 1);

		__uaccess_end();

		return ret;

	case 2:__put_user_asm(*(u16 *)src, (u16 __user *)dst,

	case 2:

		__uaccess_begin();

		__put_user_asm(*(u16 *)src, (u16 __user *)dst,

			      ret, "w", "w", "ir", 2);

		__uaccess_end();

		return ret;

	case 4:__put_user_asm(*(u32 *)src, (u32 __user *)dst,

	case 4:

		__uaccess_begin();

		__put_user_asm(*(u32 *)src, (u32 __user *)dst,

			      ret, "l", "k", "ir", 4);

		__uaccess_end();

		return ret;

	case 8:__put_user_asm(*(u64 *)src, (u64 __user *)dst,

	case 8:

		__uaccess_begin();

		__put_user_asm(*(u64 *)src, (u64 __user *)dst,

			      ret, "q", "", "er", 8);

		__uaccess_end();

		return ret;

	case 10:

		__uaccess_begin();

		__put_user_asm(*(u64 *)src, (u64 __user *)dst,

			       ret, "q", "", "er", 10);

		if (unlikely(ret))

			return ret;

		if (likely(!ret)) {

			asm("":::"memory");

			__put_user_asm(4[(u16 *)src], 4 + (u16 __user *)dst,

				       ret, "w", "w", "ir", 2);

		}

		__uaccess_end();

		return ret;

	case 16:

		__uaccess_begin();

		__put_user_asm(*(u64 *)src, (u64 __user *)dst,

			       ret, "q", "", "er", 16);

		if (unlikely(ret))

			return ret;

		if (likely(!ret)) {

			asm("":::"memory");

			__put_user_asm(1[(u64 *)src], 1 + (u64 __user *)dst,

				       ret, "q", "", "er", 8);

		}

		__uaccess_end();

		return ret;

	default:

		return copy_user_generic((__force void *)dst, src, size);
@@ -160,39 +190,47 @@ int __copy_in_user(void __user *dst, const void __user *src, unsigned size) 
	switch (size) {

	case 1: {

		u8 tmp;

		__uaccess_begin();

		__get_user_asm(tmp, (u8 __user *)src,

			       ret, "b", "b", "=q", 1);

		if (likely(!ret))

			__put_user_asm(tmp, (u8 __user *)dst,

				       ret, "b", "b", "iq", 1);

		__uaccess_end();

		return ret;

	}

	case 2: {

		u16 tmp;

		__uaccess_begin();

		__get_user_asm(tmp, (u16 __user *)src,

			       ret, "w", "w", "=r", 2);

		if (likely(!ret))

			__put_user_asm(tmp, (u16 __user *)dst,

				       ret, "w", "w", "ir", 2);

		__uaccess_end();

		return ret;

	}



	case 4: {

		u32 tmp;

		__uaccess_begin();

		__get_user_asm(tmp, (u32 __user *)src,

			       ret, "l", "k", "=r", 4);

		if (likely(!ret))

			__put_user_asm(tmp, (u32 __user *)dst,

				       ret, "l", "k", "ir", 4);

		__uaccess_end();

		return ret;

	}

	case 8: {

		u64 tmp;

		__uaccess_begin();

		__get_user_asm(tmp, (u64 __user *)src,

			       ret, "q", "", "=r", 8);

		if (likely(!ret))

			__put_user_asm(tmp, (u64 __user *)dst,

				       ret, "q", "", "er", 8);

		__uaccess_end();

		return ret;

	}

	default:

CRA Git | Maintained and supported by SUSTech CRA and CCSE