Commit bac4e960 authored by Russell King's avatar Russell King Committed by Russell King
Browse files

[ARM] barriers: improve xchg, bitops and atomic SMP barriers 



Mathieu Desnoyers pointed out that the ARM barriers were lacking:

- cmpxchg, xchg and atomic add return need memory barriers on
  architectures which can reorder the relative order in which memory
  read/writes can be seen between CPUs, which seems to include recent
  ARM architectures. Those barriers are currently missing on ARM.

- test_and_xxx_bit were missing SMP barriers.

So put these barriers in.  Provide separate atomic_add/atomic_sub
operations which do not require barriers.

Reported-Reviewed-and-Acked-by: default avatarMathieu Desnoyers <mathieu.desnoyers@polymtl.ca>
Signed-off-by: default avatarRussell King <rmk+kernel@arm.linux.org.uk>
parent 29081571

arch/arm/include/asm/assembler.h

+13 −0
Original line number Diff line number Diff line
@@ -114,3 +114,16 @@ 
	.align	3;				\

	.long	9999b,9001f;			\

	.previous



/*

 * SMP data memory barrier

 */

	.macro	smp_dmb

#ifdef CONFIG_SMP

#if __LINUX_ARM_ARCH__ >= 7

	dmb

#elif __LINUX_ARM_ARCH__ == 6

	mcr	p15, 0, r0, c7, c10, 5	@ dmb

#endif

#endif

	.endm

arch/arm/include/asm/atomic.h

+52 −9
Original line number Diff line number Diff line
@@ -44,11 +44,29 @@ static inline void atomic_set(atomic_t *v, int i) 
	: "cc");

}



static inline void atomic_add(int i, atomic_t *v)

{

	unsigned long tmp;

	int result;



	__asm__ __volatile__("@ atomic_add\n"

"1:	ldrex	%0, [%2]\n"

"	add	%0, %0, %3\n"

"	strex	%1, %0, [%2]\n"

"	teq	%1, #0\n"

"	bne	1b"

	: "=&r" (result), "=&r" (tmp)

	: "r" (&v->counter), "Ir" (i)

	: "cc");

}



static inline int atomic_add_return(int i, atomic_t *v)

{

	unsigned long tmp;

	int result;



	smp_mb();



	__asm__ __volatile__("@ atomic_add_return\n"

"1:	ldrex	%0, [%2]\n"

"	add	%0, %0, %3\n"
@@ -59,14 +77,34 @@ static inline int atomic_add_return(int i, atomic_t *v) 
	: "r" (&v->counter), "Ir" (i)

	: "cc");



	smp_mb();



	return result;

}



static inline void atomic_sub(int i, atomic_t *v)

{

	unsigned long tmp;

	int result;



	__asm__ __volatile__("@ atomic_sub\n"

"1:	ldrex	%0, [%2]\n"

"	sub	%0, %0, %3\n"

"	strex	%1, %0, [%2]\n"

"	teq	%1, #0\n"

"	bne	1b"

	: "=&r" (result), "=&r" (tmp)

	: "r" (&v->counter), "Ir" (i)

	: "cc");

}



static inline int atomic_sub_return(int i, atomic_t *v)

{

	unsigned long tmp;

	int result;



	smp_mb();



	__asm__ __volatile__("@ atomic_sub_return\n"

"1:	ldrex	%0, [%2]\n"

"	sub	%0, %0, %3\n"
@@ -77,6 +115,8 @@ static inline int atomic_sub_return(int i, atomic_t *v) 
	: "r" (&v->counter), "Ir" (i)

	: "cc");



	smp_mb();



	return result;

}
@@ -84,6 +124,8 @@ static inline int atomic_cmpxchg(atomic_t *ptr, int old, int new) 
{

	unsigned long oldval, res;



	smp_mb();



	do {

		__asm__ __volatile__("@ atomic_cmpxchg\n"

		"ldrex	%1, [%2]\n"
@@ -95,6 +137,8 @@ static inline int atomic_cmpxchg(atomic_t *ptr, int old, int new) 
		    : "cc");

	} while (res);



	smp_mb();



	return oldval;

}
@@ -135,6 +179,7 @@ static inline int atomic_add_return(int i, atomic_t *v) 


	return val;

}

#define atomic_add(i, v)	(void) atomic_add_return(i, v)



static inline int atomic_sub_return(int i, atomic_t *v)

{
@@ -148,6 +193,7 @@ static inline int atomic_sub_return(int i, atomic_t *v) 


	return val;

}

#define atomic_sub(i, v)	(void) atomic_sub_return(i, v)



static inline int atomic_cmpxchg(atomic_t *v, int old, int new)

{
@@ -187,10 +233,8 @@ static inline int atomic_add_unless(atomic_t *v, int a, int u) 
}

#define atomic_inc_not_zero(v) atomic_add_unless((v), 1, 0)



#define atomic_add(i, v)	(void) atomic_add_return(i, v)

#define atomic_inc(v)		(void) atomic_add_return(1, v)

#define atomic_sub(i, v)	(void) atomic_sub_return(i, v)

#define atomic_dec(v)		(void) atomic_sub_return(1, v)

#define atomic_inc(v)		atomic_add(1, v)

#define atomic_dec(v)		atomic_sub(1, v)



#define atomic_inc_and_test(v)	(atomic_add_return(1, v) == 0)

#define atomic_dec_and_test(v)	(atomic_sub_return(1, v) == 0)
@@ -200,11 +244,10 @@ static inline int atomic_add_unless(atomic_t *v, int a, int u) 


#define atomic_add_negative(i,v) (atomic_add_return(i, v) < 0)



/* Atomic operations are already serializing on ARM */

#define smp_mb__before_atomic_dec()	barrier()

#define smp_mb__after_atomic_dec()	barrier()

#define smp_mb__before_atomic_inc()	barrier()

#define smp_mb__after_atomic_inc()	barrier()

#define smp_mb__before_atomic_dec()	smp_mb()

#define smp_mb__after_atomic_dec()	smp_mb()

#define smp_mb__before_atomic_inc()	smp_mb()

#define smp_mb__after_atomic_inc()	smp_mb()



#include <asm-generic/atomic.h>

#endif

arch/arm/include/asm/system.h

+3 −0
Original line number Diff line number Diff line
@@ -248,6 +248,8 @@ static inline unsigned long __xchg(unsigned long x, volatile void *ptr, int size 
	unsigned int tmp;

#endif



	smp_mb();



	switch (size) {

#if __LINUX_ARM_ARCH__ >= 6

	case 1:
@@ -307,6 +309,7 @@ static inline unsigned long __xchg(unsigned long x, volatile void *ptr, int size 
		__bad_xchg(ptr, size), ret = 0;

		break;

	}

	smp_mb();



	return ret;

}

arch/arm/kernel/entry-armv.S

+1 −4
Original line number Diff line number Diff line
@@ -815,10 +815,7 @@ __kuser_helper_start: 
 */



__kuser_memory_barrier:				@ 0xffff0fa0



#if __LINUX_ARM_ARCH__ >= 6 && defined(CONFIG_SMP)

	mcr	p15, 0, r0, c7, c10, 5	@ dmb

#endif

	smp_dmb

	usr_ret	lr



	.align	5

arch/arm/lib/bitops.h

+2 −0
Original line number Diff line number Diff line
@@ -18,12 +18,14 @@ 
	mov	r2, #1

	add	r1, r1, r0, lsr #3	@ Get byte offset

	mov	r3, r2, lsl r3		@ create mask

	smp_dmb

1:	ldrexb	r2, [r1]

	ands	r0, r2, r3		@ save old value of bit

	\instr	r2, r2, r3			@ toggle bit

	strexb	ip, r2, [r1]

	cmp	ip, #0

	bne	1b

	smp_dmb

	cmp	r0, #0

	movne	r0, #1

2:	mov	pc, lr

CRA Git | Maintained and supported by SUSTech CRA and CCSE