Commit 79e1dd05 authored by Arjan van de Ven's avatar Arjan van de Ven Committed by Ingo Molnar
Browse files

x86: Provide an alternative() based cmpxchg64() 



cmpxchg64() today generates, to quote Linus, "barf bag" code.

cmpxchg64() is about to get used in the scheduler to fix a bug there,
but it's a prerequisite that cmpxchg64() first be made non-sucking.

This patch turns cmpxchg64() into an efficient implementation that
uses the alternative() mechanism to just use the raw instruction on
all modern systems.

Note: the fallback is NOT smp safe, just like the current fallback
is not SMP safe. (Interested parties with i486 based SMP systems
are welcome to submit fix patches for that.)

Signed-off-by: default avatarArjan van de Ven <arjan@linux.intel.com>
Acked-by: default avatarLinus Torvalds <torvalds@linux-foundation.org>
[ fixed asm constraint bug ]
Fixed-by: default avatarEric Dumazet <eric.dumazet@gmail.com>
Cc: Martin Schwidefsky <schwidefsky@de.ibm.com>
Cc: John Stultz <johnstul@us.ibm.com>
Cc: Peter Zijlstra <a.p.zijlstra@chello.nl>
LKML-Reference: <20090930170754.0886ff2e@infradead.org>
Signed-off-by: default avatarIngo Molnar <mingo@elte.hu>
parent 17d857be

arch/x86/include/asm/cmpxchg_32.h

+17 −13
Original line number Diff line number Diff line
@@ -315,16 +315,20 @@ extern unsigned long long cmpxchg_486_u64(volatile void *, u64, u64); 
#define cmpxchg64(ptr, o, n)					\

({								\

	__typeof__(*(ptr)) __ret;				\

	if (likely(boot_cpu_data.x86 > 4))				\

		__ret = (__typeof__(*(ptr)))__cmpxchg64((ptr),		\

				(unsigned long long)(o),		\

				(unsigned long long)(n));		\

	else								\

		__ret = (__typeof__(*(ptr)))cmpxchg_486_u64((ptr),	\

				(unsigned long long)(o),		\

				(unsigned long long)(n));		\

	__ret;								\

})

	__typeof__(*(ptr)) __old = (o);				\

	__typeof__(*(ptr)) __new = (n);				\

	alternative_io("call cmpxchg8b_emu",			\

			"lock; cmpxchg8b (%%esi)" ,		\

		       X86_FEATURE_CX8,				\

		       "=A" (__ret),				\

		       "S" ((ptr)), "0" (__old),		\

		       "b" ((unsigned int)__new),		\

		       "c" ((unsigned int)(__new>>32))		\

		       : "memory");				\

	__ret; })







#define cmpxchg64_local(ptr, o, n)					\

({									\

	__typeof__(*(ptr)) __ret;					\

arch/x86/kernel/i386_ksyms_32.c

+8 −0
Original line number Diff line number Diff line
@@ -10,6 +10,14 @@ 
EXPORT_SYMBOL(mcount);

#endif



/*

 * Note, this is a prototype to get at the symbol for

 * the export, but dont use it from C code, it is used

 * by assembly code and is not using C calling convention!

 */

extern void cmpxchg8b_emu(void);

EXPORT_SYMBOL(cmpxchg8b_emu);



/* Networking helper routines. */

EXPORT_SYMBOL(csum_partial_copy_generic);

arch/x86/lib/Makefile

+1 −1
Original line number Diff line number Diff line
@@ -15,7 +15,7 @@ ifeq ($(CONFIG_X86_32),y) 
        obj-y += atomic64_32.o

        lib-y += checksum_32.o

        lib-y += strstr_32.o

        lib-y += semaphore_32.o string_32.o

        lib-y += semaphore_32.o string_32.o cmpxchg8b_emu.o



        lib-$(CONFIG_X86_USE_3DNOW) += mmx_32.o

else

arch/x86/lib/cmpxchg8b_emu.S

0 → 100644
+57 −0
Original line number Diff line number Diff line 
/*

 *	This program is free software; you can redistribute it and/or

 *	modify it under the terms of the GNU General Public License

 *	as published by the Free Software Foundation; version 2

 *	of the License.

 *

 */



#include <linux/linkage.h>

#include <asm/alternative-asm.h>

#include <asm/frame.h>

#include <asm/dwarf2.h>





.text



/*

 * Inputs:

 * %esi : memory location to compare

 * %eax : low 32 bits of old value

 * %edx : high 32 bits of old value

 * %ebx : low 32 bits of new value

 * %ecx : high 32 bits of new value

 */

ENTRY(cmpxchg8b_emu)

CFI_STARTPROC



#

# Emulate 'cmpxchg8b (%esi)' on UP except we don't

# set the whole ZF thing (caller will just compare

# eax:edx with the expected value)

#

cmpxchg8b_emu:

	pushfl

	cli



	cmpl  (%esi), %eax

	jne not_same

	cmpl 4(%esi), %edx

	jne half_same



	movl %ebx,  (%esi)

	movl %ecx, 4(%esi)



	popfl

	ret



 not_same:

	movl  (%esi), %eax

 half_same:

	movl 4(%esi), %edx



	popfl

	ret



CFI_ENDPROC

ENDPROC(cmpxchg8b_emu)

CRA Git | Maintained and supported by SUSTech CRA and CCSE