Commit f10e2e5b authored by Anton Blanchard's avatar Anton Blanchard Committed by Benjamin Herrenschmidt
Browse files

powerpc: Rename LWSYNC_ON_SMP to PPC_RELEASE_BARRIER, ISYNC_ON_SMP to PPC_ACQUIRE_BARRIER 



For performance reasons we are about to change ISYNC_ON_SMP to sometimes be
lwsync. Now that the macro name doesn't make sense, change it and LWSYNC_ON_SMP
to better explain what the barriers are doing.

Signed-off-by: default avatarAnton Blanchard <anton@samba.org>
Signed-off-by: default avatarBenjamin Herrenschmidt <benh@kernel.crashing.org>
parent 66d99b88

arch/powerpc/include/asm/atomic.h

+24 −24
Original line number Diff line number Diff line
@@ -49,13 +49,13 @@ static __inline__ int atomic_add_return(int a, atomic_t *v) 
	int t;



	__asm__ __volatile__(

	LWSYNC_ON_SMP

	PPC_RELEASE_BARRIER

"1:	lwarx	%0,0,%2		# atomic_add_return\n\

	add	%0,%1,%0\n"

	PPC405_ERR77(0,%2)

"	stwcx.	%0,0,%2 \n\

	bne-	1b"

	ISYNC_ON_SMP

	PPC_ACQUIRE_BARRIER

	: "=&r" (t)

	: "r" (a), "r" (&v->counter)

	: "cc", "memory");
@@ -85,13 +85,13 @@ static __inline__ int atomic_sub_return(int a, atomic_t *v) 
	int t;



	__asm__ __volatile__(

	LWSYNC_ON_SMP

	PPC_RELEASE_BARRIER

"1:	lwarx	%0,0,%2		# atomic_sub_return\n\

	subf	%0,%1,%0\n"

	PPC405_ERR77(0,%2)

"	stwcx.	%0,0,%2 \n\

	bne-	1b"

	ISYNC_ON_SMP

	PPC_ACQUIRE_BARRIER

	: "=&r" (t)

	: "r" (a), "r" (&v->counter)

	: "cc", "memory");
@@ -119,13 +119,13 @@ static __inline__ int atomic_inc_return(atomic_t *v) 
	int t;



	__asm__ __volatile__(

	LWSYNC_ON_SMP

	PPC_RELEASE_BARRIER

"1:	lwarx	%0,0,%1		# atomic_inc_return\n\

	addic	%0,%0,1\n"

	PPC405_ERR77(0,%1)

"	stwcx.	%0,0,%1 \n\

	bne-	1b"

	ISYNC_ON_SMP

	PPC_ACQUIRE_BARRIER

	: "=&r" (t)

	: "r" (&v->counter)

	: "cc", "xer", "memory");
@@ -163,13 +163,13 @@ static __inline__ int atomic_dec_return(atomic_t *v) 
	int t;



	__asm__ __volatile__(

	LWSYNC_ON_SMP

	PPC_RELEASE_BARRIER

"1:	lwarx	%0,0,%1		# atomic_dec_return\n\

	addic	%0,%0,-1\n"

	PPC405_ERR77(0,%1)

"	stwcx.	%0,0,%1\n\

	bne-	1b"

	ISYNC_ON_SMP

	PPC_ACQUIRE_BARRIER

	: "=&r" (t)

	: "r" (&v->counter)

	: "cc", "xer", "memory");
@@ -194,7 +194,7 @@ static __inline__ int atomic_add_unless(atomic_t *v, int a, int u) 
	int t;



	__asm__ __volatile__ (

	LWSYNC_ON_SMP

	PPC_RELEASE_BARRIER

"1:	lwarx	%0,0,%1		# atomic_add_unless\n\

	cmpw	0,%0,%3 \n\

	beq-	2f \n\
@@ -202,7 +202,7 @@ static __inline__ int atomic_add_unless(atomic_t *v, int a, int u) 
	PPC405_ERR77(0,%2)

"	stwcx.	%0,0,%1 \n\

	bne-	1b \n"

	ISYNC_ON_SMP

	PPC_ACQUIRE_BARRIER

"	subf	%0,%2,%0 \n\

2:"

	: "=&r" (t)
@@ -227,7 +227,7 @@ static __inline__ int atomic_dec_if_positive(atomic_t *v) 
	int t;



	__asm__ __volatile__(

	LWSYNC_ON_SMP

	PPC_RELEASE_BARRIER

"1:	lwarx	%0,0,%1		# atomic_dec_if_positive\n\

	cmpwi	%0,1\n\

	addi	%0,%0,-1\n\
@@ -235,7 +235,7 @@ static __inline__ int atomic_dec_if_positive(atomic_t *v) 
	PPC405_ERR77(0,%1)

"	stwcx.	%0,0,%1\n\

	bne-	1b"

	ISYNC_ON_SMP

	PPC_ACQUIRE_BARRIER

	"\n\

2:"	: "=&b" (t)

	: "r" (&v->counter)
@@ -286,12 +286,12 @@ static __inline__ long atomic64_add_return(long a, atomic64_t *v) 
	long t;



	__asm__ __volatile__(

	LWSYNC_ON_SMP

	PPC_RELEASE_BARRIER

"1:	ldarx	%0,0,%2		# atomic64_add_return\n\

	add	%0,%1,%0\n\

	stdcx.	%0,0,%2 \n\

	bne-	1b"

	ISYNC_ON_SMP

	PPC_ACQUIRE_BARRIER

	: "=&r" (t)

	: "r" (a), "r" (&v->counter)

	: "cc", "memory");
@@ -320,12 +320,12 @@ static __inline__ long atomic64_sub_return(long a, atomic64_t *v) 
	long t;



	__asm__ __volatile__(

	LWSYNC_ON_SMP

	PPC_RELEASE_BARRIER

"1:	ldarx	%0,0,%2		# atomic64_sub_return\n\

	subf	%0,%1,%0\n\

	stdcx.	%0,0,%2 \n\

	bne-	1b"

	ISYNC_ON_SMP

	PPC_ACQUIRE_BARRIER

	: "=&r" (t)

	: "r" (a), "r" (&v->counter)

	: "cc", "memory");
@@ -352,12 +352,12 @@ static __inline__ long atomic64_inc_return(atomic64_t *v) 
	long t;



	__asm__ __volatile__(

	LWSYNC_ON_SMP

	PPC_RELEASE_BARRIER

"1:	ldarx	%0,0,%1		# atomic64_inc_return\n\

	addic	%0,%0,1\n\

	stdcx.	%0,0,%1 \n\

	bne-	1b"

	ISYNC_ON_SMP

	PPC_ACQUIRE_BARRIER

	: "=&r" (t)

	: "r" (&v->counter)

	: "cc", "xer", "memory");
@@ -394,12 +394,12 @@ static __inline__ long atomic64_dec_return(atomic64_t *v) 
	long t;



	__asm__ __volatile__(

	LWSYNC_ON_SMP

	PPC_RELEASE_BARRIER

"1:	ldarx	%0,0,%1		# atomic64_dec_return\n\

	addic	%0,%0,-1\n\

	stdcx.	%0,0,%1\n\

	bne-	1b"

	ISYNC_ON_SMP

	PPC_ACQUIRE_BARRIER

	: "=&r" (t)

	: "r" (&v->counter)

	: "cc", "xer", "memory");
@@ -419,13 +419,13 @@ static __inline__ long atomic64_dec_if_positive(atomic64_t *v) 
	long t;



	__asm__ __volatile__(

	LWSYNC_ON_SMP

	PPC_RELEASE_BARRIER

"1:	ldarx	%0,0,%1		# atomic64_dec_if_positive\n\

	addic.	%0,%0,-1\n\

	blt-	2f\n\

	stdcx.	%0,0,%1\n\

	bne-	1b"

	ISYNC_ON_SMP

	PPC_ACQUIRE_BARRIER

	"\n\

2:"	: "=&r" (t)

	: "r" (&v->counter)
@@ -451,14 +451,14 @@ static __inline__ int atomic64_add_unless(atomic64_t *v, long a, long u) 
	long t;



	__asm__ __volatile__ (

	LWSYNC_ON_SMP

	PPC_RELEASE_BARRIER

"1:	ldarx	%0,0,%1		# atomic_add_unless\n\

	cmpd	0,%0,%3 \n\

	beq-	2f \n\

	add	%0,%2,%0 \n"

"	stdcx.	%0,0,%1 \n\

	bne-	1b \n"

	ISYNC_ON_SMP

	PPC_ACQUIRE_BARRIER

"	subf	%0,%2,%0 \n\

2:"

	: "=&r" (t)

arch/powerpc/include/asm/bitops.h

+10 −6
Original line number Diff line number Diff line
@@ -78,7 +78,7 @@ static __inline__ void fn(unsigned long mask, \ 


DEFINE_BITOP(set_bits, or, "", "")

DEFINE_BITOP(clear_bits, andc, "", "")

DEFINE_BITOP(clear_bits_unlock, andc, LWSYNC_ON_SMP, "")

DEFINE_BITOP(clear_bits_unlock, andc, PPC_RELEASE_BARRIER, "")

DEFINE_BITOP(change_bits, xor, "", "")



static __inline__ void set_bit(int nr, volatile unsigned long *addr)
@@ -124,10 +124,14 @@ static __inline__ unsigned long fn( \ 
	return (old & mask);				\

}



DEFINE_TESTOP(test_and_set_bits, or, LWSYNC_ON_SMP, ISYNC_ON_SMP, 0)

DEFINE_TESTOP(test_and_set_bits_lock, or, "", ISYNC_ON_SMP, 1)

DEFINE_TESTOP(test_and_clear_bits, andc, LWSYNC_ON_SMP, ISYNC_ON_SMP, 0)

DEFINE_TESTOP(test_and_change_bits, xor, LWSYNC_ON_SMP, ISYNC_ON_SMP, 0)

DEFINE_TESTOP(test_and_set_bits, or, PPC_RELEASE_BARRIER,

	      PPC_ACQUIRE_BARRIER, 0)

DEFINE_TESTOP(test_and_set_bits_lock, or, "",

	      PPC_ACQUIRE_BARRIER, 1)

DEFINE_TESTOP(test_and_clear_bits, andc, PPC_RELEASE_BARRIER,

	      PPC_ACQUIRE_BARRIER, 0)

DEFINE_TESTOP(test_and_change_bits, xor, PPC_RELEASE_BARRIER,

	      PPC_ACQUIRE_BARRIER, 0)



static __inline__ int test_and_set_bit(unsigned long nr,

				       volatile unsigned long *addr)
@@ -158,7 +162,7 @@ static __inline__ int test_and_change_bit(unsigned long nr, 


static __inline__ void __clear_bit_unlock(int nr, volatile unsigned long *addr)

{

	__asm__ __volatile__(LWSYNC_ON_SMP "" ::: "memory");

	__asm__ __volatile__(PPC_RELEASE_BARRIER "" ::: "memory");

	__clear_bit(nr, addr);

}

arch/powerpc/include/asm/futex.h

+3 −3
Original line number Diff line number Diff line
@@ -11,7 +11,7 @@ 


#define __futex_atomic_op(insn, ret, oldval, uaddr, oparg) \

  __asm__ __volatile ( \

	LWSYNC_ON_SMP \

	PPC_RELEASE_BARRIER \

"1:	lwarx	%0,0,%2\n" \

	insn \

	PPC405_ERR77(0, %2) \
@@ -90,14 +90,14 @@ futex_atomic_cmpxchg_inatomic(int __user *uaddr, int oldval, int newval) 
		return -EFAULT;



        __asm__ __volatile__ (

        LWSYNC_ON_SMP

        PPC_RELEASE_BARRIER

"1:     lwarx   %0,0,%2         # futex_atomic_cmpxchg_inatomic\n\

        cmpw    0,%0,%3\n\

        bne-    3f\n"

        PPC405_ERR77(0,%2)

"2:     stwcx.  %4,0,%2\n\

        bne-    1b\n"

        ISYNC_ON_SMP

        PPC_ACQUIRE_BARRIER

"3:	.section .fixup,\"ax\"\n\

4:	li	%0,%5\n\

	b	3b\n\

arch/powerpc/include/asm/mutex.h

+3 −3
Original line number Diff line number Diff line
@@ -15,7 +15,7 @@ static inline int __mutex_cmpxchg_lock(atomic_t *v, int old, int new) 
	PPC405_ERR77(0,%1)

"	stwcx.	%3,0,%1\n\

	bne-	1b"

	ISYNC_ON_SMP

	PPC_ACQUIRE_BARRIER

	"\n\

2:"

	: "=&r" (t)
@@ -35,7 +35,7 @@ static inline int __mutex_dec_return_lock(atomic_t *v) 
	PPC405_ERR77(0,%1)

"	stwcx.	%0,0,%1\n\

	bne-	1b"

	ISYNC_ON_SMP

	PPC_ACQUIRE_BARRIER

	: "=&r" (t)

	: "r" (&v->counter)

	: "cc", "memory");
@@ -48,7 +48,7 @@ static inline int __mutex_inc_return_unlock(atomic_t *v) 
	int t;



	__asm__ __volatile__(

	LWSYNC_ON_SMP

	PPC_RELEASE_BARRIER

"1:	lwarx	%0,0,%1		# mutex unlock\n\

	addic	%0,%0,1\n"

	PPC405_ERR77(0,%1)

arch/powerpc/include/asm/spinlock.h

+13 −12
Original line number Diff line number Diff line
@@ -65,9 +65,10 @@ static inline unsigned long __arch_spin_trylock(arch_spinlock_t *lock) 
	cmpwi		0,%0,0\n\

	bne-		2f\n\

	stwcx.		%1,0,%2\n\

	bne-		1b\n\

	isync\n\

2:"	: "=&r" (tmp)

	bne-		1b\n"

	PPC_ACQUIRE_BARRIER

"2:"

	: "=&r" (tmp)

	: "r" (token), "r" (&lock->slock)

	: "cr0", "memory");
@@ -145,7 +146,7 @@ static inline void arch_spin_unlock(arch_spinlock_t *lock) 
{

	SYNC_IO;

	__asm__ __volatile__("# arch_spin_unlock\n\t"

				LWSYNC_ON_SMP: : :"memory");

				PPC_RELEASE_BARRIER: : :"memory");

	lock->slock = 0;

}
@@ -193,9 +194,9 @@ static inline long __arch_read_trylock(arch_rwlock_t *rw) 
	ble-		2f\n"

	PPC405_ERR77(0,%1)

"	stwcx.		%0,0,%1\n\

	bne-		1b\n\

	isync\n\

2:"	: "=&r" (tmp)

	bne-		1b\n"

	PPC_ACQUIRE_BARRIER

"2:"	: "=&r" (tmp)

	: "r" (&rw->lock)

	: "cr0", "xer", "memory");
@@ -217,9 +218,9 @@ static inline long __arch_write_trylock(arch_rwlock_t *rw) 
	bne-		2f\n"

	PPC405_ERR77(0,%1)

"	stwcx.		%1,0,%2\n\

	bne-		1b\n\

	isync\n\

2:"	: "=&r" (tmp)

	bne-		1b\n"

	PPC_ACQUIRE_BARRIER

"2:"	: "=&r" (tmp)

	: "r" (token), "r" (&rw->lock)

	: "cr0", "memory");
@@ -270,7 +271,7 @@ static inline void arch_read_unlock(arch_rwlock_t *rw) 


	__asm__ __volatile__(

	"# read_unlock\n\t"

	LWSYNC_ON_SMP

	PPC_RELEASE_BARRIER

"1:	lwarx		%0,0,%1\n\

	addic		%0,%0,-1\n"

	PPC405_ERR77(0,%1)
@@ -284,7 +285,7 @@ static inline void arch_read_unlock(arch_rwlock_t *rw) 
static inline void arch_write_unlock(arch_rwlock_t *rw)

{

	__asm__ __volatile__("# write_unlock\n\t"

				LWSYNC_ON_SMP: : :"memory");

				PPC_RELEASE_BARRIER: : :"memory");

	rw->lock = 0;

}

CRA Git | Maintained and supported by SUSTech CRA and CCSE