Commit 4f3316c2 authored by Peter Zijlstra's avatar Peter Zijlstra Committed by Ingo Molnar
Browse files

locking,arch,sparc: Fold atomic_ops 



Many of the atomic op implementations are the same except for one
instruction; fold the lot into a few CPP macros and reduce LoC.

This also prepares for easy addition of new ops.

Signed-off-by: default avatarPeter Zijlstra <peterz@infradead.org>
Acked-by: default avatarDavid S. Miller <davem@davemloft.net>
Cc: Bjorn Helgaas <bhelgaas@google.com>
Cc: Kirill Tkhai <tkhai@yandex.ru>
Cc: Linus Torvalds <torvalds@linux-foundation.org>
Cc: Paul E. McKenney <paulmck@linux.vnet.ibm.com>
Cc: Sam Ravnborg <sam@ravnborg.org>
Cc: sparclinux@vger.kernel.org
Link: http://lkml.kernel.org/r/20140508135852.825281379@infradead.org


Signed-off-by: default avatarIngo Molnar <mingo@kernel.org>
parent c6470150

arch/sparc/include/asm/atomic_32.h

+8 −9
Original line number Diff line number Diff line
@@ -20,7 +20,7 @@ 


#define ATOMIC_INIT(i)  { (i) }



int __atomic_add_return(int, atomic_t *);

int atomic_add_return(int, atomic_t *);

int atomic_cmpxchg(atomic_t *, int, int);

#define atomic_xchg(v, new) (xchg(&((v)->counter), new))

int __atomic_add_unless(atomic_t *, int, int);
@@ -28,15 +28,14 @@ void atomic_set(atomic_t *, int); 


#define atomic_read(v)          (*(volatile int *)&(v)->counter)



#define atomic_add(i, v)	((void)__atomic_add_return( (int)(i), (v)))

#define atomic_sub(i, v)	((void)__atomic_add_return(-(int)(i), (v)))

#define atomic_inc(v)		((void)__atomic_add_return(        1, (v)))

#define atomic_dec(v)		((void)__atomic_add_return(       -1, (v)))

#define atomic_add(i, v)	((void)atomic_add_return( (int)(i), (v)))

#define atomic_sub(i, v)	((void)atomic_add_return(-(int)(i), (v)))

#define atomic_inc(v)		((void)atomic_add_return(        1, (v)))

#define atomic_dec(v)		((void)atomic_add_return(       -1, (v)))



#define atomic_add_return(i, v)	(__atomic_add_return( (int)(i), (v)))

#define atomic_sub_return(i, v)	(__atomic_add_return(-(int)(i), (v)))

#define atomic_inc_return(v)	(__atomic_add_return(        1, (v)))

#define atomic_dec_return(v)	(__atomic_add_return(       -1, (v)))

#define atomic_sub_return(i, v)	(atomic_add_return(-(int)(i), (v)))

#define atomic_inc_return(v)	(atomic_add_return(        1, (v)))

#define atomic_dec_return(v)	(atomic_add_return(       -1, (v)))



#define atomic_add_negative(a, v)	(atomic_add_return((a), (v)) < 0)

arch/sparc/include/asm/atomic_64.h

+23 −22
Original line number Diff line number Diff line
@@ -20,27 +20,28 @@ 
#define atomic_set(v, i)	(((v)->counter) = i)

#define atomic64_set(v, i)	(((v)->counter) = i)



void atomic_add(int, atomic_t *);

void atomic64_add(long, atomic64_t *);

void atomic_sub(int, atomic_t *);

void atomic64_sub(long, atomic64_t *);

#define ATOMIC_OP(op)							\

void atomic_##op(int, atomic_t *);					\

void atomic64_##op(long, atomic64_t *);



int atomic_add_ret(int, atomic_t *);

long atomic64_add_ret(long, atomic64_t *);

int atomic_sub_ret(int, atomic_t *);

long atomic64_sub_ret(long, atomic64_t *);

#define ATOMIC_OP_RETURN(op)						\

int atomic_##op##_return(int, atomic_t *);				\

long atomic64_##op##_return(long, atomic64_t *);



#define atomic_dec_return(v) atomic_sub_ret(1, v)

#define atomic64_dec_return(v) atomic64_sub_ret(1, v)

#define ATOMIC_OPS(op) ATOMIC_OP(op) ATOMIC_OP_RETURN(op)



#define atomic_inc_return(v) atomic_add_ret(1, v)

#define atomic64_inc_return(v) atomic64_add_ret(1, v)

ATOMIC_OPS(add)

ATOMIC_OPS(sub)



#define atomic_sub_return(i, v) atomic_sub_ret(i, v)

#define atomic64_sub_return(i, v) atomic64_sub_ret(i, v)

#undef ATOMIC_OPS

#undef ATOMIC_OP_RETURN

#undef ATOMIC_OP



#define atomic_add_return(i, v) atomic_add_ret(i, v)

#define atomic64_add_return(i, v) atomic64_add_ret(i, v)

#define atomic_dec_return(v)   atomic_sub_return(1, v)

#define atomic64_dec_return(v) atomic64_sub_return(1, v)



#define atomic_inc_return(v)   atomic_add_return(1, v)

#define atomic64_inc_return(v) atomic64_add_return(1, v)



/*

 * atomic_inc_and_test - increment and test
@@ -53,11 +54,11 @@ long atomic64_sub_ret(long, atomic64_t *); 
#define atomic_inc_and_test(v) (atomic_inc_return(v) == 0)

#define atomic64_inc_and_test(v) (atomic64_inc_return(v) == 0)



#define atomic_sub_and_test(i, v) (atomic_sub_ret(i, v) == 0)

#define atomic64_sub_and_test(i, v) (atomic64_sub_ret(i, v) == 0)

#define atomic_sub_and_test(i, v) (atomic_sub_return(i, v) == 0)

#define atomic64_sub_and_test(i, v) (atomic64_sub_return(i, v) == 0)



#define atomic_dec_and_test(v) (atomic_sub_ret(1, v) == 0)

#define atomic64_dec_and_test(v) (atomic64_sub_ret(1, v) == 0)

#define atomic_dec_and_test(v) (atomic_sub_return(1, v) == 0)

#define atomic64_dec_and_test(v) (atomic64_sub_return(1, v) == 0)



#define atomic_inc(v) atomic_add(1, v)

#define atomic64_inc(v) atomic64_add(1, v)
@@ -65,8 +66,8 @@ long atomic64_sub_ret(long, atomic64_t *); 
#define atomic_dec(v) atomic_sub(1, v)

#define atomic64_dec(v) atomic64_sub(1, v)



#define atomic_add_negative(i, v) (atomic_add_ret(i, v) < 0)

#define atomic64_add_negative(i, v) (atomic64_add_ret(i, v) < 0)

#define atomic_add_negative(i, v) (atomic_add_return(i, v) < 0)

#define atomic64_add_negative(i, v) (atomic64_add_return(i, v) < 0)



#define atomic_cmpxchg(v, o, n) (cmpxchg(&((v)->counter), (o), (n)))

#define atomic_xchg(v, new) (xchg(&((v)->counter), new))

arch/sparc/kernel/smp_64.c

+1 −1
Original line number Diff line number Diff line
@@ -1138,7 +1138,7 @@ static unsigned long penguins_are_doing_time; 


void smp_capture(void)

{

	int result = atomic_add_ret(1, &smp_capture_depth);

	int result = atomic_add_return(1, &smp_capture_depth);



	if (result == 1) {

		int ncpus = num_online_cpus();

arch/sparc/lib/atomic32.c

+17 −12
Original line number Diff line number Diff line
@@ -27,18 +27,23 @@ static DEFINE_SPINLOCK(dummy); 


#endif /* SMP */



int __atomic_add_return(int i, atomic_t *v)

{

	int ret;

	unsigned long flags;

	spin_lock_irqsave(ATOMIC_HASH(v), flags);



	ret = (v->counter += i);



	spin_unlock_irqrestore(ATOMIC_HASH(v), flags);

	return ret;

}

EXPORT_SYMBOL(__atomic_add_return);

#define ATOMIC_OP(op, cop)						\

int atomic_##op##_return(int i, atomic_t *v)				\

{									\

	int ret;							\

	unsigned long flags;						\

	spin_lock_irqsave(ATOMIC_HASH(v), flags);			\

									\

	ret = (v->counter cop i);					\

									\

	spin_unlock_irqrestore(ATOMIC_HASH(v), flags);			\

	return ret;							\

}									\

EXPORT_SYMBOL(atomic_##op##_return);



ATOMIC_OP(add, +=)



#undef ATOMIC_OP



int atomic_cmpxchg(atomic_t *v, int old, int new)

{

arch/sparc/lib/atomic_64.S

+67 −96
Original line number Diff line number Diff line
@@ -14,109 +14,80 @@ 
	 * memory barriers, and a second which returns

	 * a value and does the barriers.

	 */

ENTRY(atomic_add) /* %o0 = increment, %o1 = atomic_ptr */

	BACKOFF_SETUP(%o2)

1:	lduw	[%o1], %g1

	add	%g1, %o0, %g7

	cas	[%o1], %g1, %g7

	cmp	%g1, %g7

	bne,pn	%icc, BACKOFF_LABEL(2f, 1b)

	 nop

	retl

	 nop

2:	BACKOFF_SPIN(%o2, %o3, 1b)

ENDPROC(atomic_add)



ENTRY(atomic_sub) /* %o0 = decrement, %o1 = atomic_ptr */

	BACKOFF_SETUP(%o2)

1:	lduw	[%o1], %g1

	sub	%g1, %o0, %g7

	cas	[%o1], %g1, %g7

	cmp	%g1, %g7

	bne,pn	%icc, BACKOFF_LABEL(2f, 1b)

	 nop

	retl

	 nop

2:	BACKOFF_SPIN(%o2, %o3, 1b)

ENDPROC(atomic_sub)

#define ATOMIC_OP(op)							\

ENTRY(atomic_##op) /* %o0 = increment, %o1 = atomic_ptr */		\

	BACKOFF_SETUP(%o2);						\

1:	lduw	[%o1], %g1;						\

	op	%g1, %o0, %g7;						\

	cas	[%o1], %g1, %g7;					\

	cmp	%g1, %g7;						\

	bne,pn	%icc, BACKOFF_LABEL(2f, 1b);				\

	 nop;								\

	retl;								\

	 nop;								\

2:	BACKOFF_SPIN(%o2, %o3, 1b);					\

ENDPROC(atomic_##op);							\



ENTRY(atomic_add_ret) /* %o0 = increment, %o1 = atomic_ptr */

	BACKOFF_SETUP(%o2)

1:	lduw	[%o1], %g1

	add	%g1, %o0, %g7

	cas	[%o1], %g1, %g7

	cmp	%g1, %g7

	bne,pn	%icc, BACKOFF_LABEL(2f, 1b)

	 add	%g1, %o0, %g1

	retl

	 sra	%g1, 0, %o0

2:	BACKOFF_SPIN(%o2, %o3, 1b)

ENDPROC(atomic_add_ret)

#define ATOMIC_OP_RETURN(op)						\

ENTRY(atomic_##op##_return) /* %o0 = increment, %o1 = atomic_ptr */	\

	BACKOFF_SETUP(%o2);						\

1:	lduw	[%o1], %g1;						\

	op	%g1, %o0, %g7;						\

	cas	[%o1], %g1, %g7;					\

	cmp	%g1, %g7;						\

	bne,pn	%icc, BACKOFF_LABEL(2f, 1b);				\

	 add	%g1, %o0, %g1;						\

	retl;								\

	 sra	%g1, 0, %o0;						\

2:	BACKOFF_SPIN(%o2, %o3, 1b);					\

ENDPROC(atomic_##op##_return);



ENTRY(atomic_sub_ret) /* %o0 = decrement, %o1 = atomic_ptr */

	BACKOFF_SETUP(%o2)

1:	lduw	[%o1], %g1

	sub	%g1, %o0, %g7

	cas	[%o1], %g1, %g7

	cmp	%g1, %g7

	bne,pn	%icc, BACKOFF_LABEL(2f, 1b)

	 sub	%g1, %o0, %g1

	retl

	 sra	%g1, 0, %o0

2:	BACKOFF_SPIN(%o2, %o3, 1b)

ENDPROC(atomic_sub_ret)

#define ATOMIC_OPS(op) ATOMIC_OP(op) ATOMIC_OP_RETURN(op)



ENTRY(atomic64_add) /* %o0 = increment, %o1 = atomic_ptr */

	BACKOFF_SETUP(%o2)

1:	ldx	[%o1], %g1

	add	%g1, %o0, %g7

	casx	[%o1], %g1, %g7

	cmp	%g1, %g7

	bne,pn	%xcc, BACKOFF_LABEL(2f, 1b)

	 nop

	retl

	 nop

2:	BACKOFF_SPIN(%o2, %o3, 1b)

ENDPROC(atomic64_add)

ATOMIC_OPS(add)

ATOMIC_OPS(sub)



ENTRY(atomic64_sub) /* %o0 = decrement, %o1 = atomic_ptr */

	BACKOFF_SETUP(%o2)

1:	ldx	[%o1], %g1

	sub	%g1, %o0, %g7

	casx	[%o1], %g1, %g7

	cmp	%g1, %g7

	bne,pn	%xcc, BACKOFF_LABEL(2f, 1b)

	 nop

	retl

	 nop

2:	BACKOFF_SPIN(%o2, %o3, 1b)

ENDPROC(atomic64_sub)

#undef ATOMIC_OPS

#undef ATOMIC_OP_RETURN

#undef ATOMIC_OP



ENTRY(atomic64_add_ret) /* %o0 = increment, %o1 = atomic_ptr */

	BACKOFF_SETUP(%o2)

1:	ldx	[%o1], %g1

	add	%g1, %o0, %g7

	casx	[%o1], %g1, %g7

	cmp	%g1, %g7

	bne,pn	%xcc, BACKOFF_LABEL(2f, 1b)

	 nop

	retl

	 add	%g1, %o0, %o0

2:	BACKOFF_SPIN(%o2, %o3, 1b)

ENDPROC(atomic64_add_ret)

#define ATOMIC64_OP(op)							\

ENTRY(atomic64_##op) /* %o0 = increment, %o1 = atomic_ptr */		\

	BACKOFF_SETUP(%o2);						\

1:	ldx	[%o1], %g1;						\

	op	%g1, %o0, %g7;						\

	casx	[%o1], %g1, %g7;					\

	cmp	%g1, %g7;						\

	bne,pn	%xcc, BACKOFF_LABEL(2f, 1b);				\

	 nop;								\

	retl;								\

	 nop;								\

2:	BACKOFF_SPIN(%o2, %o3, 1b);					\

ENDPROC(atomic64_##op);							\



ENTRY(atomic64_sub_ret) /* %o0 = decrement, %o1 = atomic_ptr */

	BACKOFF_SETUP(%o2)

1:	ldx	[%o1], %g1

	sub	%g1, %o0, %g7

	casx	[%o1], %g1, %g7

	cmp	%g1, %g7

	bne,pn	%xcc, BACKOFF_LABEL(2f, 1b)

	 nop

	retl

	 sub	%g1, %o0, %o0

2:	BACKOFF_SPIN(%o2, %o3, 1b)

ENDPROC(atomic64_sub_ret)

#define ATOMIC64_OP_RETURN(op)						\

ENTRY(atomic64_##op##_return) /* %o0 = increment, %o1 = atomic_ptr */	\

	BACKOFF_SETUP(%o2);						\

1:	ldx	[%o1], %g1;						\

	op	%g1, %o0, %g7;						\

	casx	[%o1], %g1, %g7;					\

	cmp	%g1, %g7;						\

	bne,pn	%xcc, BACKOFF_LABEL(2f, 1b);				\

	 nop;								\

	retl;								\

	 add	%g1, %o0, %o0;						\

2:	BACKOFF_SPIN(%o2, %o3, 1b);					\

ENDPROC(atomic64_##op##_return);



#define ATOMIC64_OPS(op) ATOMIC64_OP(op) ATOMIC64_OP_RETURN(op)



ATOMIC64_OPS(add)

ATOMIC64_OPS(sub)



#undef ATOMIC64_OPS

#undef ATOMIC64_OP_RETURN

#undef ATOMIC64_OP



ENTRY(atomic64_dec_if_positive) /* %o0 = atomic_ptr */

	BACKOFF_SETUP(%o2)

CRA Git | Maintained and supported by SUSTech CRA and CCSE