Commit db38ee87 authored by Will Deacon's avatar Will Deacon Committed by Russell King
Browse files

ARM: 7983/1: atomics: implement a better __atomic_add_unless for v6+ 



Looking at perf profiles of multi-threaded hackbench runs, a significant
performance hit appears to manifest from the cmpxchg loop used to
implement the 32-bit atomic_add_unless function. This can be mitigated
by writing a direct implementation of __atomic_add_unless which doesn't
require iteration outside of the atomic operation.

Signed-off-by: default avatarWill Deacon <will.deacon@arm.com>
Signed-off-by: default avatarRussell King <rmk+kernel@arm.linux.org.uk>
parent d98b90ea

arch/arm/include/asm/atomic.h

+31 −4
Original line number Diff line number Diff line
@@ -141,6 +141,33 @@ static inline int atomic_cmpxchg(atomic_t *ptr, int old, int new) 
	return oldval;

}



static inline int __atomic_add_unless(atomic_t *v, int a, int u)

{

	int oldval, newval;

	unsigned long tmp;



	smp_mb();

	prefetchw(&v->counter);



	__asm__ __volatile__ ("@ atomic_add_unless\n"

"1:	ldrex	%0, [%4]\n"

"	teq	%0, %5\n"

"	beq	2f\n"

"	add	%1, %0, %6\n"

"	strex	%2, %1, [%4]\n"

"	teq	%2, #0\n"

"	bne	1b\n"

"2:"

	: "=&r" (oldval), "=&r" (newval), "=&r" (tmp), "+Qo" (v->counter)

	: "r" (&v->counter), "r" (u), "r" (a)

	: "cc");



	if (oldval != u)

		smp_mb();



	return oldval;

}



#else /* ARM_ARCH_6 */



#ifdef CONFIG_SMP
@@ -189,10 +216,6 @@ static inline int atomic_cmpxchg(atomic_t *v, int old, int new) 
	return ret;

}



#endif /* __LINUX_ARM_ARCH__ */



#define atomic_xchg(v, new) (xchg(&((v)->counter), new))



static inline int __atomic_add_unless(atomic_t *v, int a, int u)

{

	int c, old;
@@ -203,6 +226,10 @@ static inline int __atomic_add_unless(atomic_t *v, int a, int u) 
	return c;

}



#endif /* __LINUX_ARM_ARCH__ */



#define atomic_xchg(v, new) (xchg(&((v)->counter), new))



#define atomic_inc(v)		atomic_add(1, v)

#define atomic_dec(v)		atomic_sub(1, v)

CRA Git | Maintained and supported by SUSTech CRA and CCSE