Commit 69734b64 authored by Linus Torvalds's avatar Linus Torvalds
Browse files

Merge branch 'x86-asm-for-linus' of git://git.kernel.org/pub/scm/linux/kernel/git/tip/tip 

* 'x86-asm-for-linus' of git://git.kernel.org/pub/scm/linux/kernel/git/tip/tip: (21 commits)
  x86: Fix atomic64_xxx_cx8() functions
  x86: Fix and improve cmpxchg_double{,_local}()
  x86_64, asm: Optimise fls(), ffs() and fls64()
  x86, bitops: Move fls64.h inside __KERNEL__
  x86: Fix and improve percpu_cmpxchg{8,16}b_double()
  x86: Report cpb and eff_freq_ro flags correctly
  x86/i386: Use less assembly in strlen(), speed things up a bit
  x86: Use the same node_distance for 32 and 64-bit
  x86: Fix rflags in FAKE_STACK_FRAME
  x86: Clean up and extend do_int3()
  x86: Call do_notify_resume() with interrupts enabled
  x86/div64: Add a micro-optimization shortcut if base is power of two
  x86-64: Cleanup some assembly entry points
  x86-64: Slightly shorten line system call entry and exit paths
  x86-64: Reduce amount of redundant code generated for invalidate_interruptNN
  x86-64: Slightly shorten int_ret_from_sys_call
  x86, efi: Convert efi_phys_get_time() args to physical addresses
  x86: Default to vsyscall=emulate
  x86-64: Set siginfo and context on vsyscall emulation faults
  x86: consolidate xchg and xadd macros
  ...
parents 67b02431 ceb7b40b

Documentation/kernel-parameters.txt

+3 −4
Original line number Diff line number Diff line
@@ -2755,11 +2755,10 @@ bytes respectively. Such letter suffixes can also be entirely omitted. 
			functions are at fixed addresses, they make nice

			targets for exploits that can control RIP.



			emulate     Vsyscalls turn into traps and are emulated

			            reasonably safely.

			emulate     [default] Vsyscalls turn into traps and are

			            emulated reasonably safely.



			native      [default] Vsyscalls are native syscall

			            instructions.

			native      Vsyscalls are native syscall instructions.

			            This is a little bit faster than trapping

			            and makes a few dynamic recompilers work

			            better than they would in emulation mode.

arch/x86/ia32/ia32entry.S

+19 −24
Original line number Diff line number Diff line
@@ -134,7 +134,7 @@ ENTRY(ia32_sysenter_target) 
	CFI_REL_OFFSET rsp,0

	pushfq_cfi

	/*CFI_REL_OFFSET rflags,0*/

	movl	8*3-THREAD_SIZE+TI_sysenter_return(%rsp), %r10d

	movl	TI_sysenter_return+THREAD_INFO(%rsp,3*8-KERNEL_STACK_OFFSET),%r10d

	CFI_REGISTER rip,r10

	pushq_cfi $__USER32_CS

	/*CFI_REL_OFFSET cs,0*/
@@ -150,9 +150,8 @@ ENTRY(ia32_sysenter_target) 
 	.section __ex_table,"a"

 	.quad 1b,ia32_badarg

 	.previous	

	GET_THREAD_INFO(%r10)

	orl    $TS_COMPAT,TI_status(%r10)

	testl  $_TIF_WORK_SYSCALL_ENTRY,TI_flags(%r10)

	orl     $TS_COMPAT,TI_status+THREAD_INFO(%rsp,RIP-ARGOFFSET)

	testl   $_TIF_WORK_SYSCALL_ENTRY,TI_flags+THREAD_INFO(%rsp,RIP-ARGOFFSET)

	CFI_REMEMBER_STATE

	jnz  sysenter_tracesys

	cmpq	$(IA32_NR_syscalls-1),%rax
@@ -162,13 +161,12 @@ sysenter_do_call: 
sysenter_dispatch:

	call	*ia32_sys_call_table(,%rax,8)

	movq	%rax,RAX-ARGOFFSET(%rsp)

	GET_THREAD_INFO(%r10)

	DISABLE_INTERRUPTS(CLBR_NONE)

	TRACE_IRQS_OFF

	testl	$_TIF_ALLWORK_MASK,TI_flags(%r10)

	testl	$_TIF_ALLWORK_MASK,TI_flags+THREAD_INFO(%rsp,RIP-ARGOFFSET)

	jnz	sysexit_audit

sysexit_from_sys_call:

	andl    $~TS_COMPAT,TI_status(%r10)

	andl    $~TS_COMPAT,TI_status+THREAD_INFO(%rsp,RIP-ARGOFFSET)

	/* clear IF, that popfq doesn't enable interrupts early */

	andl  $~0x200,EFLAGS-R11(%rsp) 

	movl	RIP-R11(%rsp),%edx		/* User %eip */
@@ -205,7 +203,7 @@ sysexit_from_sys_call: 
	.endm



	.macro auditsys_exit exit

	testl $(_TIF_ALLWORK_MASK & ~_TIF_SYSCALL_AUDIT),TI_flags(%r10)

	testl $(_TIF_ALLWORK_MASK & ~_TIF_SYSCALL_AUDIT),TI_flags+THREAD_INFO(%rsp,RIP-ARGOFFSET)

	jnz ia32_ret_from_sys_call

	TRACE_IRQS_ON

	sti
@@ -215,12 +213,11 @@ sysexit_from_sys_call: 
	movzbl %al,%edi		/* zero-extend that into %edi */

	inc %edi /* first arg, 0->1(AUDITSC_SUCCESS), 1->2(AUDITSC_FAILURE) */

	call audit_syscall_exit

	GET_THREAD_INFO(%r10)

	movl RAX-ARGOFFSET(%rsp),%eax	/* reload syscall return value */

	movl $(_TIF_ALLWORK_MASK & ~_TIF_SYSCALL_AUDIT),%edi

	cli

	TRACE_IRQS_OFF

	testl %edi,TI_flags(%r10)

	testl %edi,TI_flags+THREAD_INFO(%rsp,RIP-ARGOFFSET)

	jz \exit

	CLEAR_RREGS -ARGOFFSET

	jmp int_with_check
@@ -238,7 +235,7 @@ sysexit_audit: 


sysenter_tracesys:

#ifdef CONFIG_AUDITSYSCALL

	testl	$(_TIF_WORK_SYSCALL_ENTRY & ~_TIF_SYSCALL_AUDIT),TI_flags(%r10)

	testl	$(_TIF_WORK_SYSCALL_ENTRY & ~_TIF_SYSCALL_AUDIT),TI_flags+THREAD_INFO(%rsp,RIP-ARGOFFSET)

	jz	sysenter_auditsys

#endif

	SAVE_REST
@@ -309,9 +306,8 @@ ENTRY(ia32_cstar_target) 
	.section __ex_table,"a"

	.quad 1b,ia32_badarg

	.previous	

	GET_THREAD_INFO(%r10)

	orl   $TS_COMPAT,TI_status(%r10)

	testl $_TIF_WORK_SYSCALL_ENTRY,TI_flags(%r10)

	orl     $TS_COMPAT,TI_status+THREAD_INFO(%rsp,RIP-ARGOFFSET)

	testl   $_TIF_WORK_SYSCALL_ENTRY,TI_flags+THREAD_INFO(%rsp,RIP-ARGOFFSET)

	CFI_REMEMBER_STATE

	jnz   cstar_tracesys

	cmpq $IA32_NR_syscalls-1,%rax
@@ -321,13 +317,12 @@ cstar_do_call: 
cstar_dispatch:

	call *ia32_sys_call_table(,%rax,8)

	movq %rax,RAX-ARGOFFSET(%rsp)

	GET_THREAD_INFO(%r10)

	DISABLE_INTERRUPTS(CLBR_NONE)

	TRACE_IRQS_OFF

	testl $_TIF_ALLWORK_MASK,TI_flags(%r10)

	testl $_TIF_ALLWORK_MASK,TI_flags+THREAD_INFO(%rsp,RIP-ARGOFFSET)

	jnz sysretl_audit

sysretl_from_sys_call:

	andl $~TS_COMPAT,TI_status(%r10)

	andl $~TS_COMPAT,TI_status+THREAD_INFO(%rsp,RIP-ARGOFFSET)

	RESTORE_ARGS 0,-ARG_SKIP,0,0,0

	movl RIP-ARGOFFSET(%rsp),%ecx

	CFI_REGISTER rip,rcx
@@ -355,7 +350,7 @@ sysretl_audit: 


cstar_tracesys:

#ifdef CONFIG_AUDITSYSCALL

	testl $(_TIF_WORK_SYSCALL_ENTRY & ~_TIF_SYSCALL_AUDIT),TI_flags(%r10)

	testl $(_TIF_WORK_SYSCALL_ENTRY & ~_TIF_SYSCALL_AUDIT),TI_flags+THREAD_INFO(%rsp,RIP-ARGOFFSET)

	jz cstar_auditsys

#endif

	xchgl %r9d,%ebp
@@ -420,9 +415,8 @@ ENTRY(ia32_syscall) 
	/* note the registers are not zero extended to the sf.

	   this could be a problem. */

	SAVE_ARGS 0,1,0

	GET_THREAD_INFO(%r10)

	orl   $TS_COMPAT,TI_status(%r10)

	testl $_TIF_WORK_SYSCALL_ENTRY,TI_flags(%r10)

	orl $TS_COMPAT,TI_status+THREAD_INFO(%rsp,RIP-ARGOFFSET)

	testl $_TIF_WORK_SYSCALL_ENTRY,TI_flags+THREAD_INFO(%rsp,RIP-ARGOFFSET)

	jnz ia32_tracesys

	cmpq $(IA32_NR_syscalls-1),%rax

	ja ia32_badsys
@@ -459,8 +453,8 @@ quiet_ni_syscall: 
	CFI_ENDPROC

	

	.macro PTREGSCALL label, func, arg

	.globl \label

\label:

	ALIGN

GLOBAL(\label)

	leaq \func(%rip),%rax

	leaq -ARGOFFSET+8(%rsp),\arg	/* 8 for return address */

	jmp  ia32_ptregs_common
@@ -477,7 +471,8 @@ quiet_ni_syscall: 
	PTREGSCALL stub32_vfork, sys_vfork, %rdi

	PTREGSCALL stub32_iopl, sys_iopl, %rsi



ENTRY(ia32_ptregs_common)

	ALIGN

ia32_ptregs_common:

	popq %r11

	CFI_ENDPROC

	CFI_STARTPROC32	simple

arch/x86/include/asm/alternative-asm.h

+2 −2
Original line number Diff line number Diff line
@@ -4,10 +4,10 @@ 


#ifdef CONFIG_SMP

	.macro LOCK_PREFIX

1:	lock

672:	lock

	.section .smp_locks,"a"

	.balign 4

	.long 1b - .

	.long 672b - .

	.previous

	.endm

#else

arch/x86/include/asm/bitops.h

+63 −13
Original line number Diff line number Diff line
@@ -380,6 +380,8 @@ static inline unsigned long __fls(unsigned long word) 
	return word;

}



#undef ADDR



#ifdef __KERNEL__

/**

 * ffs - find first set bit in word
@@ -395,10 +397,25 @@ static inline unsigned long __fls(unsigned long word) 
static inline int ffs(int x)

{

	int r;

#ifdef CONFIG_X86_CMOV



#ifdef CONFIG_X86_64

	/*

	 * AMD64 says BSFL won't clobber the dest reg if x==0; Intel64 says the

	 * dest reg is undefined if x==0, but their CPU architect says its

	 * value is written to set it to the same as before, except that the

	 * top 32 bits will be cleared.

	 *

	 * We cannot do this on 32 bits because at the very least some

	 * 486 CPUs did not behave this way.

	 */

	long tmp = -1;

	asm("bsfl %1,%0"

	    : "=r" (r)

	    : "rm" (x), "0" (tmp));

#elif defined(CONFIG_X86_CMOV)

	asm("bsfl %1,%0\n\t"

	    "cmovzl %2,%0"

	    : "=r" (r) : "rm" (x), "r" (-1));

	    : "=&r" (r) : "rm" (x), "r" (-1));

#else

	asm("bsfl %1,%0\n\t"

	    "jnz 1f\n\t"
@@ -422,7 +439,22 @@ static inline int ffs(int x) 
static inline int fls(int x)

{

	int r;

#ifdef CONFIG_X86_CMOV



#ifdef CONFIG_X86_64

	/*

	 * AMD64 says BSRL won't clobber the dest reg if x==0; Intel64 says the

	 * dest reg is undefined if x==0, but their CPU architect says its

	 * value is written to set it to the same as before, except that the

	 * top 32 bits will be cleared.

	 *

	 * We cannot do this on 32 bits because at the very least some

	 * 486 CPUs did not behave this way.

	 */

	long tmp = -1;

	asm("bsrl %1,%0"

	    : "=r" (r)

	    : "rm" (x), "0" (tmp));

#elif defined(CONFIG_X86_CMOV)

	asm("bsrl %1,%0\n\t"

	    "cmovzl %2,%0"

	    : "=&r" (r) : "rm" (x), "rm" (-1));
@@ -434,11 +466,35 @@ static inline int fls(int x) 
#endif

	return r + 1;

}

#endif /* __KERNEL__ */



#undef ADDR



#ifdef __KERNEL__

/**

 * fls64 - find last set bit in a 64-bit word

 * @x: the word to search

 *

 * This is defined in a similar way as the libc and compiler builtin

 * ffsll, but returns the position of the most significant set bit.

 *

 * fls64(value) returns 0 if value is 0 or the position of the last

 * set bit if value is nonzero. The last (most significant) bit is

 * at position 64.

 */

#ifdef CONFIG_X86_64

static __always_inline int fls64(__u64 x)

{

	long bitpos = -1;

	/*

	 * AMD64 says BSRQ won't clobber the dest reg if x==0; Intel64 says the

	 * dest reg is undefined if x==0, but their CPU architect says its

	 * value is written to set it to the same as before.

	 */

	asm("bsrq %1,%0"

	    : "+r" (bitpos)

	    : "rm" (x));

	return bitpos + 1;

}

#else

#include <asm-generic/bitops/fls64.h>

#endif



#include <asm-generic/bitops/find.h>
@@ -450,12 +506,6 @@ static inline int fls(int x) 


#include <asm-generic/bitops/const_hweight.h>



#endif /* __KERNEL__ */



#include <asm-generic/bitops/fls64.h>



#ifdef __KERNEL__



#include <asm-generic/bitops/le.h>



#include <asm-generic/bitops/ext2-atomic-setbit.h>

arch/x86/include/asm/cmpxchg.h

+93 −70
Original line number Diff line number Diff line
@@ -14,6 +14,8 @@ extern void __cmpxchg_wrong_size(void) 
	__compiletime_error("Bad argument size for cmpxchg");

extern void __xadd_wrong_size(void)

	__compiletime_error("Bad argument size for xadd");

extern void __add_wrong_size(void)

	__compiletime_error("Bad argument size for add");



/*

 * Constants for operation sizes. On 32-bit, the 64-bit size it set to
@@ -32,59 +34,46 @@ extern void __xadd_wrong_size(void) 
#endif



/* 

 * Note: no "lock" prefix even on SMP: xchg always implies lock anyway.

 * Since this is generally used to protect other memory information, we

 * use "asm volatile" and "memory" clobbers to prevent gcc from moving

 * information around.

 * An exchange-type operation, which takes a value and a pointer, and

 * returns a the old value.

 */

#define __xchg(x, ptr, size)						\

#define __xchg_op(ptr, arg, op, lock)					\

	({								\

	__typeof(*(ptr)) __x = (x);					\

	switch (size) {							\

	        __typeof__ (*(ptr)) __ret = (arg);			\

		switch (sizeof(*(ptr))) {				\

		case __X86_CASE_B:					\

	{								\

		volatile u8 *__ptr = (volatile u8 *)(ptr);		\

		asm volatile("xchgb %0,%1"				\

			     : "=q" (__x), "+m" (*__ptr)		\

			     : "0" (__x)				\

			     : "memory");				\

			asm volatile (lock #op "b %b0, %1\n"		\

				      : "+r" (__ret), "+m" (*(ptr))	\

				      : : "memory", "cc");		\

			break;						\

	}								\

		case __X86_CASE_W:					\

	{								\

		volatile u16 *__ptr = (volatile u16 *)(ptr);		\

		asm volatile("xchgw %0,%1"				\

			     : "=r" (__x), "+m" (*__ptr)		\

			     : "0" (__x)				\

			     : "memory");				\

			asm volatile (lock #op "w %w0, %1\n"		\

				      : "+r" (__ret), "+m" (*(ptr))	\

				      : : "memory", "cc");		\

			break;						\

	}								\

		case __X86_CASE_L:					\

	{								\

		volatile u32 *__ptr = (volatile u32 *)(ptr);		\

		asm volatile("xchgl %0,%1"				\

			     : "=r" (__x), "+m" (*__ptr)		\

			     : "0" (__x)				\

			     : "memory");				\

			asm volatile (lock #op "l %0, %1\n"		\

				      : "+r" (__ret), "+m" (*(ptr))	\

				      : : "memory", "cc");		\

			break;						\

	}								\

		case __X86_CASE_Q:					\

	{								\

		volatile u64 *__ptr = (volatile u64 *)(ptr);		\

		asm volatile("xchgq %0,%1"				\

			     : "=r" (__x), "+m" (*__ptr)		\

			     : "0" (__x)				\

			     : "memory");				\

			asm volatile (lock #op "q %q0, %1\n"		\

				      : "+r" (__ret), "+m" (*(ptr))	\

				      : : "memory", "cc");		\

			break;						\

	}								\

		default:						\

		__xchg_wrong_size();					\

			__ ## op ## _wrong_size();			\

		}							\

	__x;								\

		__ret;							\

	})



#define xchg(ptr, v)							\

	__xchg((v), (ptr), sizeof(*ptr))

/*

 * Note: no "lock" prefix even on SMP: xchg always implies lock anyway.

 * Since this is generally used to protect other memory information, we

 * use "asm volatile" and "memory" clobbers to prevent gcc from moving

 * information around.

 */

#define xchg(ptr, v)	__xchg_op((ptr), (v), xchg, "")



/*

 * Atomic compare and exchange.  Compare OLD with MEM, if identical,
@@ -165,46 +154,80 @@ extern void __xadd_wrong_size(void) 
	__cmpxchg_local((ptr), (old), (new), sizeof(*ptr))

#endif



#define __xadd(ptr, inc, lock)						\

/*

 * xadd() adds "inc" to "*ptr" and atomically returns the previous

 * value of "*ptr".

 *

 * xadd() is locked when multiple CPUs are online

 * xadd_sync() is always locked

 * xadd_local() is never locked

 */

#define __xadd(ptr, inc, lock)	__xchg_op((ptr), (inc), xadd, lock)

#define xadd(ptr, inc)		__xadd((ptr), (inc), LOCK_PREFIX)

#define xadd_sync(ptr, inc)	__xadd((ptr), (inc), "lock; ")

#define xadd_local(ptr, inc)	__xadd((ptr), (inc), "")



#define __add(ptr, inc, lock)						\

	({								\

	        __typeof__ (*(ptr)) __ret = (inc);			\

		switch (sizeof(*(ptr))) {				\

		case __X86_CASE_B:					\

			asm volatile (lock "xaddb %b0, %1\n"		\

				      : "+r" (__ret), "+m" (*(ptr))	\

				      : : "memory", "cc");		\

			asm volatile (lock "addb %b1, %0\n"		\

				      : "+m" (*(ptr)) : "ri" (inc)	\

				      : "memory", "cc");		\

			break;						\

		case __X86_CASE_W:					\

			asm volatile (lock "xaddw %w0, %1\n"		\

				      : "+r" (__ret), "+m" (*(ptr))	\

				      : : "memory", "cc");		\

			asm volatile (lock "addw %w1, %0\n"		\

				      : "+m" (*(ptr)) : "ri" (inc)	\

				      : "memory", "cc");		\

			break;						\

		case __X86_CASE_L:					\

			asm volatile (lock "xaddl %0, %1\n"		\

				      : "+r" (__ret), "+m" (*(ptr))	\

				      : : "memory", "cc");		\

			asm volatile (lock "addl %1, %0\n"		\

				      : "+m" (*(ptr)) : "ri" (inc)	\

				      : "memory", "cc");		\

			break;						\

		case __X86_CASE_Q:					\

			asm volatile (lock "xaddq %q0, %1\n"		\

				      : "+r" (__ret), "+m" (*(ptr))	\

				      : : "memory", "cc");		\

			asm volatile (lock "addq %1, %0\n"		\

				      : "+m" (*(ptr)) : "ri" (inc)	\

				      : "memory", "cc");		\

			break;						\

		default:						\

			__xadd_wrong_size();				\

			__add_wrong_size();				\

		}							\

		__ret;							\

	})



/*

 * xadd() adds "inc" to "*ptr" and atomically returns the previous

 * value of "*ptr".

 * add_*() adds "inc" to "*ptr"

 *

 * xadd() is locked when multiple CPUs are online

 * xadd_sync() is always locked

 * xadd_local() is never locked

 * __add() takes a lock prefix

 * add_smp() is locked when multiple CPUs are online

 * add_sync() is always locked

 */

#define xadd(ptr, inc)		__xadd((ptr), (inc), LOCK_PREFIX)

#define xadd_sync(ptr, inc)	__xadd((ptr), (inc), "lock; ")

#define xadd_local(ptr, inc)	__xadd((ptr), (inc), "")

#define add_smp(ptr, inc)	__add((ptr), (inc), LOCK_PREFIX)

#define add_sync(ptr, inc)	__add((ptr), (inc), "lock; ")



#define __cmpxchg_double(pfx, p1, p2, o1, o2, n1, n2)			\

({									\

	bool __ret;							\

	__typeof__(*(p1)) __old1 = (o1), __new1 = (n1);			\

	__typeof__(*(p2)) __old2 = (o2), __new2 = (n2);			\

	BUILD_BUG_ON(sizeof(*(p1)) != sizeof(long));			\

	BUILD_BUG_ON(sizeof(*(p2)) != sizeof(long));			\

	VM_BUG_ON((unsigned long)(p1) % (2 * sizeof(long)));		\

	VM_BUG_ON((unsigned long)((p1) + 1) != (unsigned long)(p2));	\

	asm volatile(pfx "cmpxchg%c4b %2; sete %0"			\

		     : "=a" (__ret), "+d" (__old2),			\

		       "+m" (*(p1)), "+m" (*(p2))			\

		     : "i" (2 * sizeof(long)), "a" (__old1),		\

		       "b" (__new1), "c" (__new2));			\

	__ret;								\

})



#define cmpxchg_double(p1, p2, o1, o2, n1, n2) \

	__cmpxchg_double(LOCK_PREFIX, p1, p2, o1, o2, n1, n2)



#define cmpxchg_double_local(p1, p2, o1, o2, n1, n2) \

	__cmpxchg_double(, p1, p2, o1, o2, n1, n2)



#endif	/* ASM_X86_CMPXCHG_H */

CRA Git | Maintained and supported by SUSTech CRA and CCSE