Commit 0b2fcfdb authored by Nick Piggin's avatar Nick Piggin Committed by Linus Torvalds
Browse files

[PATCH] atomic: add_unless cmpxchg optimise 



Without branch hints, the very unlikely chance of the loop repeating due to
cmpxchg failure is unrolled with gcc-4 that I have tested.

Improve this for architectures with a native cas/cmpxchg.  llsc archs
should try to implement this natively.

Signed-off-by: default avatarNick Piggin <npiggin@suse.de>
Cc: Andi Kleen <ak@muc.de>
Cc: Martin Schwidefsky <schwidefsky@de.ibm.com>
Cc: Heiko Carstens <heiko.carstens@de.ibm.com>
Cc: "David S. Miller" <davem@davemloft.net>
Cc: Roman Zippel <zippel@linux-m68k.org>
Signed-off-by: default avatarAndrew Morton <akpm@osdl.org>
Signed-off-by: default avatarLinus Torvalds <torvalds@osdl.org>
parent 713729e8

include/asm-i386/atomic.h

+7 −1
Original line number Diff line number Diff line
@@ -225,8 +225,14 @@ static __inline__ int atomic_sub_return(int i, atomic_t *v) 
({								\

	int c, old;						\

	c = atomic_read(v);					\

	while (c != (u) && (old = atomic_cmpxchg((v), c, c + (a))) != c) \

	for (;;) {						\

		if (unlikely(c == (u)))				\

			break;					\

		old = atomic_cmpxchg((v), c, c + (a));		\

		if (likely(old == c))				\

			break;					\

		c = old;					\

	}							\

	c != (u);						\

})

#define atomic_inc_not_zero(v) atomic_add_unless((v), 1, 0)

include/asm-ia64/atomic.h

+7 −1
Original line number Diff line number Diff line
@@ -95,8 +95,14 @@ ia64_atomic64_sub (__s64 i, atomic64_t *v) 
({								\

	int c, old;						\

	c = atomic_read(v);					\

	while (c != (u) && (old = atomic_cmpxchg((v), c, c + (a))) != c) \

	for (;;) {						\

		if (unlikely(c == (u)))				\

			break;					\

		old = atomic_cmpxchg((v), c, c + (a));		\

		if (likely(old == c))				\

			break;					\

		c = old;					\

	}							\

	c != (u);						\

})

#define atomic_inc_not_zero(v) atomic_add_unless((v), 1, 0)

include/asm-m68k/atomic.h

+7 −1
Original line number Diff line number Diff line
@@ -175,8 +175,14 @@ static inline void atomic_set_mask(unsigned long mask, unsigned long *v) 
({								\

	int c, old;						\

	c = atomic_read(v);					\

	while (c != (u) && (old = atomic_cmpxchg((v), c, c + (a))) != c) \

	for (;;) {						\

		if (unlikely(c == (u)))				\

			break;					\

		old = atomic_cmpxchg((v), c, c + (a));		\

		if (likely(old == c))				\

			break;					\

		c = old;					\

	}							\

	c != (u);						\

})

#define atomic_inc_not_zero(v) atomic_add_unless((v), 1, 0)

include/asm-s390/atomic.h

+14 −4
Original line number Diff line number Diff line
@@ -89,10 +89,15 @@ static __inline__ int atomic_cmpxchg(atomic_t *v, int old, int new) 
static __inline__ int atomic_add_unless(atomic_t *v, int a, int u)

{

	int c, old;



	c = atomic_read(v);

	while (c != u && (old = atomic_cmpxchg(v, c, c + a)) != c)

	for (;;) {

		if (unlikely(c == u))

			break;

		old = atomic_cmpxchg(v, c, c + a);

		if (likely(old == c))

			break;

		c = old;

	}

	return c != u;

}
@@ -167,10 +172,15 @@ static __inline__ int atomic64_add_unless(atomic64_t *v, 
					  long long a, long long u)

{

	long long c, old;



	c = atomic64_read(v);

	while (c != u && (old = atomic64_cmpxchg(v, c, c + a)) != c)

	for (;;) {

		if (unlikely(c == u))

			break;

		old = atomic64_cmpxchg(v, c, c + a);

		if (likely(old == c))

			break;

		c = old;

	}

	return c != u;

}

include/asm-sparc64/atomic.h

+8 −2
Original line number Diff line number Diff line
@@ -78,9 +78,15 @@ extern int atomic64_sub_ret(int, atomic64_t *); 
({								\

	int c, old;						\

	c = atomic_read(v);					\

	while (c != (u) && (old = atomic_cmpxchg((v), c, c + (a))) != c) \

	for (;;) {						\

		if (unlikely(c == (u)))				\

			break;					\

		old = atomic_cmpxchg((v), c, c + (a));		\

		if (likely(old == c))				\

			break;					\

		c = old;					\

	c != (u);						\

	}							\

	likely(c != (u));					\

})

#define atomic_inc_not_zero(v) atomic_add_unless((v), 1, 0)

CRA Git | Maintained and supported by SUSTech CRA and CCSE