Commit dbb885fe authored by Linus Torvalds's avatar Linus Torvalds
Browse files

Merge branch 'locking-arch-for-linus' of git://git.kernel.org/pub/scm/linux/kernel/git/tip/tip 

Pull arch atomic cleanups from Ingo Molnar:
 "This is a series kept separate from the main locking tree, which
  cleans up and improves various details in the atomics type handling:

   - Remove the unused atomic_or_long() method

   - Consolidate and compress atomic ops implementations between
     architectures, to reduce linecount and to make it easier to add new
     ops.

   - Rewrite generic atomic support to only require cmpxchg() from an
     architecture - generate all other methods from that"

* 'locking-arch-for-linus' of git://git.kernel.org/pub/scm/linux/kernel/git/tip/tip: (23 commits)
  locking,arch: Use ACCESS_ONCE() instead of cast to volatile in atomic_read()
  locking, mips: Fix atomics
  locking, sparc64: Fix atomics
  locking,arch: Rewrite generic atomic support
  locking,arch,xtensa: Fold atomic_ops
  locking,arch,sparc: Fold atomic_ops
  locking,arch,sh: Fold atomic_ops
  locking,arch,powerpc: Fold atomic_ops
  locking,arch,parisc: Fold atomic_ops
  locking,arch,mn10300: Fold atomic_ops
  locking,arch,mips: Fold atomic_ops
  locking,arch,metag: Fold atomic_ops
  locking,arch,m68k: Fold atomic_ops
  locking,arch,m32r: Fold atomic_ops
  locking,arch,ia64: Fold atomic_ops
  locking,arch,hexagon: Fold atomic_ops
  locking,arch,cris: Fold atomic_ops
  locking,arch,avr32: Fold atomic_ops
  locking,arch,arm64: Fold atomic_ops
  locking,arch,arm: Fold atomic_ops
  ...
parents d6dd50e0 2291059c

arch/alpha/include/asm/atomic.h

+82 −135
Original line number Diff line number Diff line
@@ -17,8 +17,8 @@ 
#define ATOMIC_INIT(i)		{ (i) }

#define ATOMIC64_INIT(i)	{ (i) }



#define atomic_read(v)		(*(volatile int *)&(v)->counter)

#define atomic64_read(v)	(*(volatile long *)&(v)->counter)

#define atomic_read(v)		ACCESS_ONCE((v)->counter)

#define atomic64_read(v)	ACCESS_ONCE((v)->counter)



#define atomic_set(v,i)		((v)->counter = (i))

#define atomic64_set(v,i)	((v)->counter = (i))
@@ -29,145 +29,92 @@ 
 * branch back to restart the operation.

 */



static __inline__ void atomic_add(int i, atomic_t * v)

{

	unsigned long temp;

	__asm__ __volatile__(

	"1:	ldl_l %0,%1\n"

	"	addl %0,%2,%0\n"

	"	stl_c %0,%1\n"

	"	beq %0,2f\n"

	".subsection 2\n"

	"2:	br 1b\n"

	".previous"

	:"=&r" (temp), "=m" (v->counter)

	:"Ir" (i), "m" (v->counter));

}



static __inline__ void atomic64_add(long i, atomic64_t * v)

{

	unsigned long temp;

	__asm__ __volatile__(

	"1:	ldq_l %0,%1\n"

	"	addq %0,%2,%0\n"

	"	stq_c %0,%1\n"

	"	beq %0,2f\n"

	".subsection 2\n"

	"2:	br 1b\n"

	".previous"

	:"=&r" (temp), "=m" (v->counter)

	:"Ir" (i), "m" (v->counter));

}



static __inline__ void atomic_sub(int i, atomic_t * v)

{

	unsigned long temp;

	__asm__ __volatile__(

	"1:	ldl_l %0,%1\n"

	"	subl %0,%2,%0\n"

	"	stl_c %0,%1\n"

	"	beq %0,2f\n"

	".subsection 2\n"

	"2:	br 1b\n"

	".previous"

	:"=&r" (temp), "=m" (v->counter)

	:"Ir" (i), "m" (v->counter));

#define ATOMIC_OP(op)							\

static __inline__ void atomic_##op(int i, atomic_t * v)			\

{									\

	unsigned long temp;						\

	__asm__ __volatile__(						\

	"1:	ldl_l %0,%1\n"						\

	"	" #op "l %0,%2,%0\n"					\

	"	stl_c %0,%1\n"						\

	"	beq %0,2f\n"						\

	".subsection 2\n"						\

	"2:	br 1b\n"						\

	".previous"							\

	:"=&r" (temp), "=m" (v->counter)				\

	:"Ir" (i), "m" (v->counter));					\

}									\



#define ATOMIC_OP_RETURN(op)						\

static inline int atomic_##op##_return(int i, atomic_t *v)		\

{									\

	long temp, result;						\

	smp_mb();							\

	__asm__ __volatile__(						\

	"1:	ldl_l %0,%1\n"						\

	"	" #op "l %0,%3,%2\n"					\

	"	" #op "l %0,%3,%0\n"					\

	"	stl_c %0,%1\n"						\

	"	beq %0,2f\n"						\

	".subsection 2\n"						\

	"2:	br 1b\n"						\

	".previous"							\

	:"=&r" (temp), "=m" (v->counter), "=&r" (result)		\

	:"Ir" (i), "m" (v->counter) : "memory");			\

	smp_mb();							\

	return result;							\

}



static __inline__ void atomic64_sub(long i, atomic64_t * v)

{

	unsigned long temp;

	__asm__ __volatile__(

	"1:	ldq_l %0,%1\n"

	"	subq %0,%2,%0\n"

	"	stq_c %0,%1\n"

	"	beq %0,2f\n"

	".subsection 2\n"

	"2:	br 1b\n"

	".previous"

	:"=&r" (temp), "=m" (v->counter)

	:"Ir" (i), "m" (v->counter));

}





/*

 * Same as above, but return the result value

 */

static inline int atomic_add_return(int i, atomic_t *v)

{

	long temp, result;

	smp_mb();

	__asm__ __volatile__(

	"1:	ldl_l %0,%1\n"

	"	addl %0,%3,%2\n"

	"	addl %0,%3,%0\n"

	"	stl_c %0,%1\n"

	"	beq %0,2f\n"

	".subsection 2\n"

	"2:	br 1b\n"

	".previous"

	:"=&r" (temp), "=m" (v->counter), "=&r" (result)

	:"Ir" (i), "m" (v->counter) : "memory");

	smp_mb();

	return result;

#define ATOMIC64_OP(op)							\

static __inline__ void atomic64_##op(long i, atomic64_t * v)		\

{									\

	unsigned long temp;						\

	__asm__ __volatile__(						\

	"1:	ldq_l %0,%1\n"						\

	"	" #op "q %0,%2,%0\n"					\

	"	stq_c %0,%1\n"						\

	"	beq %0,2f\n"						\

	".subsection 2\n"						\

	"2:	br 1b\n"						\

	".previous"							\

	:"=&r" (temp), "=m" (v->counter)				\

	:"Ir" (i), "m" (v->counter));					\

}									\



#define ATOMIC64_OP_RETURN(op)						\

static __inline__ long atomic64_##op##_return(long i, atomic64_t * v)	\

{									\

	long temp, result;						\

	smp_mb();							\

	__asm__ __volatile__(						\

	"1:	ldq_l %0,%1\n"						\

	"	" #op "q %0,%3,%2\n"					\

	"	" #op "q %0,%3,%0\n"					\

	"	stq_c %0,%1\n"						\

	"	beq %0,2f\n"						\

	".subsection 2\n"						\

	"2:	br 1b\n"						\

	".previous"							\

	:"=&r" (temp), "=m" (v->counter), "=&r" (result)		\

	:"Ir" (i), "m" (v->counter) : "memory");			\

	smp_mb();							\

	return result;							\

}



static __inline__ long atomic64_add_return(long i, atomic64_t * v)

{

	long temp, result;

	smp_mb();

	__asm__ __volatile__(

	"1:	ldq_l %0,%1\n"

	"	addq %0,%3,%2\n"

	"	addq %0,%3,%0\n"

	"	stq_c %0,%1\n"

	"	beq %0,2f\n"

	".subsection 2\n"

	"2:	br 1b\n"

	".previous"

	:"=&r" (temp), "=m" (v->counter), "=&r" (result)

	:"Ir" (i), "m" (v->counter) : "memory");

	smp_mb();

	return result;

}

#define ATOMIC_OPS(opg)							\

	ATOMIC_OP(opg)							\

	ATOMIC_OP_RETURN(opg)						\

	ATOMIC64_OP(opg)						\

	ATOMIC64_OP_RETURN(opg)



static __inline__ long atomic_sub_return(int i, atomic_t * v)

{

	long temp, result;

	smp_mb();

	__asm__ __volatile__(

	"1:	ldl_l %0,%1\n"

	"	subl %0,%3,%2\n"

	"	subl %0,%3,%0\n"

	"	stl_c %0,%1\n"

	"	beq %0,2f\n"

	".subsection 2\n"

	"2:	br 1b\n"

	".previous"

	:"=&r" (temp), "=m" (v->counter), "=&r" (result)

	:"Ir" (i), "m" (v->counter) : "memory");

	smp_mb();

	return result;

}

ATOMIC_OPS(add)

ATOMIC_OPS(sub)



static __inline__ long atomic64_sub_return(long i, atomic64_t * v)

{

	long temp, result;

	smp_mb();

	__asm__ __volatile__(

	"1:	ldq_l %0,%1\n"

	"	subq %0,%3,%2\n"

	"	subq %0,%3,%0\n"

	"	stq_c %0,%1\n"

	"	beq %0,2f\n"

	".subsection 2\n"

	"2:	br 1b\n"

	".previous"

	:"=&r" (temp), "=m" (v->counter), "=&r" (result)

	:"Ir" (i), "m" (v->counter) : "memory");

	smp_mb();

	return result;

}

#undef ATOMIC_OPS

#undef ATOMIC64_OP_RETURN

#undef ATOMIC64_OP

#undef ATOMIC_OP_RETURN

#undef ATOMIC_OP



#define atomic64_cmpxchg(v, old, new) (cmpxchg(&((v)->counter), old, new))

#define atomic64_xchg(v, new) (xchg(&((v)->counter), new))

arch/arc/include/asm/atomic.h

+63 −121
Original line number Diff line number Diff line
@@ -25,79 +25,36 @@ 


#define atomic_set(v, i) (((v)->counter) = (i))



static inline void atomic_add(int i, atomic_t *v)

{

	unsigned int temp;



	__asm__ __volatile__(

	"1:	llock   %0, [%1]	\n"

	"	add     %0, %0, %2	\n"

	"	scond   %0, [%1]	\n"

	"	bnz     1b		\n"

	: "=&r"(temp)	/* Early clobber, to prevent reg reuse */

	: "r"(&v->counter), "ir"(i)

	: "cc");

}



static inline void atomic_sub(int i, atomic_t *v)

{

	unsigned int temp;



	__asm__ __volatile__(

	"1:	llock   %0, [%1]	\n"

	"	sub     %0, %0, %2	\n"

	"	scond   %0, [%1]	\n"

	"	bnz     1b		\n"

	: "=&r"(temp)

	: "r"(&v->counter), "ir"(i)

	: "cc");

}



/* add and also return the new value */

static inline int atomic_add_return(int i, atomic_t *v)

{

	unsigned int temp;



	__asm__ __volatile__(

	"1:	llock   %0, [%1]	\n"

	"	add     %0, %0, %2	\n"

	"	scond   %0, [%1]	\n"

	"	bnz     1b		\n"

	: "=&r"(temp)

	: "r"(&v->counter), "ir"(i)

	: "cc");



	return temp;

}



static inline int atomic_sub_return(int i, atomic_t *v)

{

	unsigned int temp;



	__asm__ __volatile__(

	"1:	llock   %0, [%1]	\n"

	"	sub     %0, %0, %2	\n"

	"	scond   %0, [%1]	\n"

	"	bnz     1b		\n"

	: "=&r"(temp)

	: "r"(&v->counter), "ir"(i)

	: "cc");



	return temp;

}



static inline void atomic_clear_mask(unsigned long mask, unsigned long *addr)

{

	unsigned int temp;



	__asm__ __volatile__(

	"1:	llock   %0, [%1]	\n"

	"	bic     %0, %0, %2	\n"

	"	scond   %0, [%1]	\n"

	"	bnz     1b		\n"

	: "=&r"(temp)

	: "r"(addr), "ir"(mask)

	: "cc");

#define ATOMIC_OP(op, c_op, asm_op)					\

static inline void atomic_##op(int i, atomic_t *v)			\

{									\

	unsigned int temp;						\

									\

	__asm__ __volatile__(						\

	"1:	llock   %0, [%1]	\n"				\

	"	" #asm_op " %0, %0, %2	\n"				\

	"	scond   %0, [%1]	\n"				\

	"	bnz     1b		\n"				\

	: "=&r"(temp)	/* Early clobber, to prevent reg reuse */	\

	: "r"(&v->counter), "ir"(i)					\

	: "cc");							\

}									\



#define ATOMIC_OP_RETURN(op, c_op, asm_op)				\

static inline int atomic_##op##_return(int i, atomic_t *v)		\

{									\

	unsigned int temp;						\

									\

	__asm__ __volatile__(						\

	"1:	llock   %0, [%1]	\n"				\

	"	" #asm_op " %0, %0, %2	\n"				\

	"	scond   %0, [%1]	\n"				\

	"	bnz     1b		\n"				\

	: "=&r"(temp)							\

	: "r"(&v->counter), "ir"(i)					\

	: "cc");							\

									\

	return temp;							\

}



#else	/* !CONFIG_ARC_HAS_LLSC */
@@ -126,6 +83,7 @@ static inline void atomic_set(atomic_t *v, int i) 
	v->counter = i;

	atomic_ops_unlock(flags);

}



#endif



/*
@@ -133,62 +91,46 @@ static inline void atomic_set(atomic_t *v, int i) 
 * Locking would change to irq-disabling only (UP) and spinlocks (SMP)

 */



static inline void atomic_add(int i, atomic_t *v)

{

	unsigned long flags;



	atomic_ops_lock(flags);

	v->counter += i;

	atomic_ops_unlock(flags);

#define ATOMIC_OP(op, c_op, asm_op)					\

static inline void atomic_##op(int i, atomic_t *v)			\

{									\

	unsigned long flags;						\

									\

	atomic_ops_lock(flags);						\

	v->counter c_op i;						\

	atomic_ops_unlock(flags);					\

}



static inline void atomic_sub(int i, atomic_t *v)

{

	unsigned long flags;



	atomic_ops_lock(flags);

	v->counter -= i;

	atomic_ops_unlock(flags);

#define ATOMIC_OP_RETURN(op, c_op)					\

static inline int atomic_##op##_return(int i, atomic_t *v)		\

{									\

	unsigned long flags;						\

	unsigned long temp;						\

									\

	atomic_ops_lock(flags);						\

	temp = v->counter;						\

	temp c_op i;							\

	v->counter = temp;						\

	atomic_ops_unlock(flags);					\

									\

	return temp;							\

}



static inline int atomic_add_return(int i, atomic_t *v)

{

	unsigned long flags;

	unsigned long temp;



	atomic_ops_lock(flags);

	temp = v->counter;

	temp += i;

	v->counter = temp;

	atomic_ops_unlock(flags);



	return temp;

}



static inline int atomic_sub_return(int i, atomic_t *v)

{

	unsigned long flags;

	unsigned long temp;



	atomic_ops_lock(flags);

	temp = v->counter;

	temp -= i;

	v->counter = temp;

	atomic_ops_unlock(flags);

#endif /* !CONFIG_ARC_HAS_LLSC */



	return temp;

}

#define ATOMIC_OPS(op, c_op, asm_op)					\

	ATOMIC_OP(op, c_op, asm_op)					\

	ATOMIC_OP_RETURN(op, c_op, asm_op)



static inline void atomic_clear_mask(unsigned long mask, unsigned long *addr)

{

	unsigned long flags;

ATOMIC_OPS(add, +=, add)

ATOMIC_OPS(sub, -=, sub)

ATOMIC_OP(and, &=, and)



	atomic_ops_lock(flags);

	*addr &= ~mask;

	atomic_ops_unlock(flags);

}

#define atomic_clear_mask(mask, v) atomic_and(~(mask), (v))



#endif /* !CONFIG_ARC_HAS_LLSC */

#undef ATOMIC_OPS

#undef ATOMIC_OP_RETURN

#undef ATOMIC_OP



/**

 * __atomic_add_unless - add unless the number is a given value

arch/arm/include/asm/atomic.h

+124 −183
Original line number Diff line number Diff line
@@ -27,7 +27,7 @@ 
 * strex/ldrex monitor on some implementations. The reason we can use it for

 * atomic_set() is the clrex or dummy strex done on every exception return.

 */

#define atomic_read(v)	(*(volatile int *)&(v)->counter)

#define atomic_read(v)	ACCESS_ONCE((v)->counter)

#define atomic_set(v,i)	(((v)->counter) = (i))



#if __LINUX_ARM_ARCH__ >= 6
@@ -37,84 +37,47 @@ 
 * store exclusive to ensure that these are atomic.  We may loop

 * to ensure that the update happens.

 */

static inline void atomic_add(int i, atomic_t *v)

{

	unsigned long tmp;

	int result;



	prefetchw(&v->counter);

	__asm__ __volatile__("@ atomic_add\n"

"1:	ldrex	%0, [%3]\n"

"	add	%0, %0, %4\n"

"	strex	%1, %0, [%3]\n"

"	teq	%1, #0\n"

"	bne	1b"

	: "=&r" (result), "=&r" (tmp), "+Qo" (v->counter)

	: "r" (&v->counter), "Ir" (i)

	: "cc");

}



static inline int atomic_add_return(int i, atomic_t *v)

{

	unsigned long tmp;

	int result;



	smp_mb();

	prefetchw(&v->counter);



	__asm__ __volatile__("@ atomic_add_return\n"

"1:	ldrex	%0, [%3]\n"

"	add	%0, %0, %4\n"

"	strex	%1, %0, [%3]\n"

"	teq	%1, #0\n"

"	bne	1b"

	: "=&r" (result), "=&r" (tmp), "+Qo" (v->counter)

	: "r" (&v->counter), "Ir" (i)

	: "cc");



	smp_mb();



	return result;

}



static inline void atomic_sub(int i, atomic_t *v)

{

	unsigned long tmp;

	int result;



	prefetchw(&v->counter);

	__asm__ __volatile__("@ atomic_sub\n"

"1:	ldrex	%0, [%3]\n"

"	sub	%0, %0, %4\n"

"	strex	%1, %0, [%3]\n"

"	teq	%1, #0\n"

"	bne	1b"

	: "=&r" (result), "=&r" (tmp), "+Qo" (v->counter)

	: "r" (&v->counter), "Ir" (i)

	: "cc");

}



static inline int atomic_sub_return(int i, atomic_t *v)

{

	unsigned long tmp;

	int result;



	smp_mb();

	prefetchw(&v->counter);



	__asm__ __volatile__("@ atomic_sub_return\n"

"1:	ldrex	%0, [%3]\n"

"	sub	%0, %0, %4\n"

"	strex	%1, %0, [%3]\n"

"	teq	%1, #0\n"

"	bne	1b"

	: "=&r" (result), "=&r" (tmp), "+Qo" (v->counter)

	: "r" (&v->counter), "Ir" (i)

	: "cc");



	smp_mb();



	return result;

#define ATOMIC_OP(op, c_op, asm_op)					\

static inline void atomic_##op(int i, atomic_t *v)			\

{									\

	unsigned long tmp;						\

	int result;							\

									\

	prefetchw(&v->counter);						\

	__asm__ __volatile__("@ atomic_" #op "\n"			\

"1:	ldrex	%0, [%3]\n"						\

"	" #asm_op "	%0, %0, %4\n"					\

"	strex	%1, %0, [%3]\n"						\

"	teq	%1, #0\n"						\

"	bne	1b"							\

	: "=&r" (result), "=&r" (tmp), "+Qo" (v->counter)		\

	: "r" (&v->counter), "Ir" (i)					\

	: "cc");							\

}									\



#define ATOMIC_OP_RETURN(op, c_op, asm_op)				\

static inline int atomic_##op##_return(int i, atomic_t *v)		\

{									\

	unsigned long tmp;						\

	int result;							\

									\

	smp_mb();							\

	prefetchw(&v->counter);						\

									\

	__asm__ __volatile__("@ atomic_" #op "_return\n"		\

"1:	ldrex	%0, [%3]\n"						\

"	" #asm_op "	%0, %0, %4\n"					\

"	strex	%1, %0, [%3]\n"						\

"	teq	%1, #0\n"						\

"	bne	1b"							\

	: "=&r" (result), "=&r" (tmp), "+Qo" (v->counter)		\

	: "r" (&v->counter), "Ir" (i)					\

	: "cc");							\

									\

	smp_mb();							\

									\

	return result;							\

}



static inline int atomic_cmpxchg(atomic_t *ptr, int old, int new)
@@ -174,33 +137,29 @@ static inline int __atomic_add_unless(atomic_t *v, int a, int u) 
#error SMP not supported on pre-ARMv6 CPUs

#endif



static inline int atomic_add_return(int i, atomic_t *v)

{

	unsigned long flags;

	int val;



	raw_local_irq_save(flags);

	val = v->counter;

	v->counter = val += i;

	raw_local_irq_restore(flags);



	return val;

}

#define atomic_add(i, v)	(void) atomic_add_return(i, v)



static inline int atomic_sub_return(int i, atomic_t *v)

{

	unsigned long flags;

	int val;



	raw_local_irq_save(flags);

	val = v->counter;

	v->counter = val -= i;

	raw_local_irq_restore(flags);



	return val;

#define ATOMIC_OP(op, c_op, asm_op)					\

static inline void atomic_##op(int i, atomic_t *v)			\

{									\

	unsigned long flags;						\

									\

	raw_local_irq_save(flags);					\

	v->counter c_op i;						\

	raw_local_irq_restore(flags);					\

}									\



#define ATOMIC_OP_RETURN(op, c_op, asm_op)				\

static inline int atomic_##op##_return(int i, atomic_t *v)		\

{									\

	unsigned long flags;						\

	int val;							\

									\

	raw_local_irq_save(flags);					\

	v->counter c_op i;						\

	val = v->counter;						\

	raw_local_irq_restore(flags);					\

									\

	return val;							\

}

#define atomic_sub(i, v)	(void) atomic_sub_return(i, v)



static inline int atomic_cmpxchg(atomic_t *v, int old, int new)

{
@@ -228,6 +187,17 @@ static inline int __atomic_add_unless(atomic_t *v, int a, int u) 


#endif /* __LINUX_ARM_ARCH__ */



#define ATOMIC_OPS(op, c_op, asm_op)					\

	ATOMIC_OP(op, c_op, asm_op)					\

	ATOMIC_OP_RETURN(op, c_op, asm_op)



ATOMIC_OPS(add, +=, add)

ATOMIC_OPS(sub, -=, sub)



#undef ATOMIC_OPS

#undef ATOMIC_OP_RETURN

#undef ATOMIC_OP



#define atomic_xchg(v, new) (xchg(&((v)->counter), new))



#define atomic_inc(v)		atomic_add(1, v)
@@ -300,89 +270,60 @@ static inline void atomic64_set(atomic64_t *v, long long i) 
}

#endif



static inline void atomic64_add(long long i, atomic64_t *v)

{

	long long result;

	unsigned long tmp;



	prefetchw(&v->counter);

	__asm__ __volatile__("@ atomic64_add\n"

"1:	ldrexd	%0, %H0, [%3]\n"

"	adds	%Q0, %Q0, %Q4\n"

"	adc	%R0, %R0, %R4\n"

"	strexd	%1, %0, %H0, [%3]\n"

"	teq	%1, #0\n"

"	bne	1b"

	: "=&r" (result), "=&r" (tmp), "+Qo" (v->counter)

	: "r" (&v->counter), "r" (i)

	: "cc");

}



static inline long long atomic64_add_return(long long i, atomic64_t *v)

{

	long long result;

	unsigned long tmp;



	smp_mb();

	prefetchw(&v->counter);



	__asm__ __volatile__("@ atomic64_add_return\n"

"1:	ldrexd	%0, %H0, [%3]\n"

"	adds	%Q0, %Q0, %Q4\n"

"	adc	%R0, %R0, %R4\n"

"	strexd	%1, %0, %H0, [%3]\n"

"	teq	%1, #0\n"

"	bne	1b"

	: "=&r" (result), "=&r" (tmp), "+Qo" (v->counter)

	: "r" (&v->counter), "r" (i)

	: "cc");



	smp_mb();



	return result;

}



static inline void atomic64_sub(long long i, atomic64_t *v)

{

	long long result;

	unsigned long tmp;



	prefetchw(&v->counter);

	__asm__ __volatile__("@ atomic64_sub\n"

"1:	ldrexd	%0, %H0, [%3]\n"

"	subs	%Q0, %Q0, %Q4\n"

"	sbc	%R0, %R0, %R4\n"

"	strexd	%1, %0, %H0, [%3]\n"

"	teq	%1, #0\n"

"	bne	1b"

	: "=&r" (result), "=&r" (tmp), "+Qo" (v->counter)

	: "r" (&v->counter), "r" (i)

	: "cc");

#define ATOMIC64_OP(op, op1, op2)					\

static inline void atomic64_##op(long long i, atomic64_t *v)		\

{									\

	long long result;						\

	unsigned long tmp;						\

									\

	prefetchw(&v->counter);						\

	__asm__ __volatile__("@ atomic64_" #op "\n"			\

"1:	ldrexd	%0, %H0, [%3]\n"					\

"	" #op1 " %Q0, %Q0, %Q4\n"					\

"	" #op2 " %R0, %R0, %R4\n"					\

"	strexd	%1, %0, %H0, [%3]\n"					\

"	teq	%1, #0\n"						\

"	bne	1b"							\

	: "=&r" (result), "=&r" (tmp), "+Qo" (v->counter)		\

	: "r" (&v->counter), "r" (i)					\

	: "cc");							\

}									\



#define ATOMIC64_OP_RETURN(op, op1, op2)				\

static inline long long atomic64_##op##_return(long long i, atomic64_t *v) \

{									\

	long long result;						\

	unsigned long tmp;						\

									\

	smp_mb();							\

	prefetchw(&v->counter);						\

									\

	__asm__ __volatile__("@ atomic64_" #op "_return\n"		\

"1:	ldrexd	%0, %H0, [%3]\n"					\

"	" #op1 " %Q0, %Q0, %Q4\n"					\

"	" #op2 " %R0, %R0, %R4\n"					\

"	strexd	%1, %0, %H0, [%3]\n"					\

"	teq	%1, #0\n"						\

"	bne	1b"							\

	: "=&r" (result), "=&r" (tmp), "+Qo" (v->counter)		\

	: "r" (&v->counter), "r" (i)					\

	: "cc");							\

									\

	smp_mb();							\

									\

	return result;							\

}



static inline long long atomic64_sub_return(long long i, atomic64_t *v)

{

	long long result;

	unsigned long tmp;



	smp_mb();

	prefetchw(&v->counter);



	__asm__ __volatile__("@ atomic64_sub_return\n"

"1:	ldrexd	%0, %H0, [%3]\n"

"	subs	%Q0, %Q0, %Q4\n"

"	sbc	%R0, %R0, %R4\n"

"	strexd	%1, %0, %H0, [%3]\n"

"	teq	%1, #0\n"

"	bne	1b"

	: "=&r" (result), "=&r" (tmp), "+Qo" (v->counter)

	: "r" (&v->counter), "r" (i)

	: "cc");

#define ATOMIC64_OPS(op, op1, op2)					\

	ATOMIC64_OP(op, op1, op2)					\

	ATOMIC64_OP_RETURN(op, op1, op2)



	smp_mb();

ATOMIC64_OPS(add, adds, adc)

ATOMIC64_OPS(sub, subs, sbc)



	return result;

}

#undef ATOMIC64_OPS

#undef ATOMIC64_OP_RETURN

#undef ATOMIC64_OP



static inline long long atomic64_cmpxchg(atomic64_t *ptr, long long old,

					long long new)

arch/arm64/include/asm/atomic.h

+82 −119
Original line number Diff line number Diff line
@@ -35,7 +35,7 @@ 
 * strex/ldrex monitor on some implementations. The reason we can use it for

 * atomic_set() is the clrex or dummy strex done on every exception return.

 */

#define atomic_read(v)	(*(volatile int *)&(v)->counter)

#define atomic_read(v)	ACCESS_ONCE((v)->counter)

#define atomic_set(v,i)	(((v)->counter) = (i))



/*
@@ -43,69 +43,51 @@ 
 * store exclusive to ensure that these are atomic.  We may loop

 * to ensure that the update happens.

 */

static inline void atomic_add(int i, atomic_t *v)

{

	unsigned long tmp;

	int result;



	asm volatile("// atomic_add\n"

"1:	ldxr	%w0, %2\n"

"	add	%w0, %w0, %w3\n"

"	stxr	%w1, %w0, %2\n"

"	cbnz	%w1, 1b"

	: "=&r" (result), "=&r" (tmp), "+Q" (v->counter)

	: "Ir" (i));

}



static inline int atomic_add_return(int i, atomic_t *v)

{

	unsigned long tmp;

	int result;



	asm volatile("// atomic_add_return\n"

"1:	ldxr	%w0, %2\n"

"	add	%w0, %w0, %w3\n"

"	stlxr	%w1, %w0, %2\n"

"	cbnz	%w1, 1b"

	: "=&r" (result), "=&r" (tmp), "+Q" (v->counter)

	: "Ir" (i)

	: "memory");



	smp_mb();

	return result;

}



static inline void atomic_sub(int i, atomic_t *v)

{

	unsigned long tmp;

	int result;



	asm volatile("// atomic_sub\n"

"1:	ldxr	%w0, %2\n"

"	sub	%w0, %w0, %w3\n"

"	stxr	%w1, %w0, %2\n"

"	cbnz	%w1, 1b"

	: "=&r" (result), "=&r" (tmp), "+Q" (v->counter)

	: "Ir" (i));

#define ATOMIC_OP(op, asm_op)						\

static inline void atomic_##op(int i, atomic_t *v)			\

{									\

	unsigned long tmp;						\

	int result;							\

									\

	asm volatile("// atomic_" #op "\n"				\

"1:	ldxr	%w0, %2\n"						\

"	" #asm_op "	%w0, %w0, %w3\n"				\

"	stxr	%w1, %w0, %2\n"						\

"	cbnz	%w1, 1b"						\

	: "=&r" (result), "=&r" (tmp), "+Q" (v->counter)		\

	: "Ir" (i));							\

}									\



#define ATOMIC_OP_RETURN(op, asm_op)					\

static inline int atomic_##op##_return(int i, atomic_t *v)		\

{									\

	unsigned long tmp;						\

	int result;							\

									\

	asm volatile("// atomic_" #op "_return\n"			\

"1:	ldxr	%w0, %2\n"						\

"	" #asm_op "	%w0, %w0, %w3\n"				\

"	stlxr	%w1, %w0, %2\n"						\

"	cbnz	%w1, 1b"						\

	: "=&r" (result), "=&r" (tmp), "+Q" (v->counter)		\

	: "Ir" (i)							\

	: "memory");							\

									\

	smp_mb();							\

	return result;							\

}



static inline int atomic_sub_return(int i, atomic_t *v)

{

	unsigned long tmp;

	int result;

#define ATOMIC_OPS(op, asm_op)						\

	ATOMIC_OP(op, asm_op)						\

	ATOMIC_OP_RETURN(op, asm_op)



	asm volatile("// atomic_sub_return\n"

"1:	ldxr	%w0, %2\n"

"	sub	%w0, %w0, %w3\n"

"	stlxr	%w1, %w0, %2\n"

"	cbnz	%w1, 1b"

	: "=&r" (result), "=&r" (tmp), "+Q" (v->counter)

	: "Ir" (i)

	: "memory");

ATOMIC_OPS(add, add)

ATOMIC_OPS(sub, sub)



	smp_mb();

	return result;

}

#undef ATOMIC_OPS

#undef ATOMIC_OP_RETURN

#undef ATOMIC_OP



static inline int atomic_cmpxchg(atomic_t *ptr, int old, int new)

{
@@ -157,72 +139,53 @@ static inline int __atomic_add_unless(atomic_t *v, int a, int u) 
 */

#define ATOMIC64_INIT(i) { (i) }



#define atomic64_read(v)	(*(volatile long *)&(v)->counter)

#define atomic64_read(v)	ACCESS_ONCE((v)->counter)

#define atomic64_set(v,i)	(((v)->counter) = (i))



static inline void atomic64_add(u64 i, atomic64_t *v)

{

	long result;

	unsigned long tmp;



	asm volatile("// atomic64_add\n"

"1:	ldxr	%0, %2\n"

"	add	%0, %0, %3\n"

"	stxr	%w1, %0, %2\n"

"	cbnz	%w1, 1b"

	: "=&r" (result), "=&r" (tmp), "+Q" (v->counter)

	: "Ir" (i));

#define ATOMIC64_OP(op, asm_op)						\

static inline void atomic64_##op(long i, atomic64_t *v)			\

{									\

	long result;							\

	unsigned long tmp;						\

									\

	asm volatile("// atomic64_" #op "\n"				\

"1:	ldxr	%0, %2\n"						\

"	" #asm_op "	%0, %0, %3\n"					\

"	stxr	%w1, %0, %2\n"						\

"	cbnz	%w1, 1b"						\

	: "=&r" (result), "=&r" (tmp), "+Q" (v->counter)		\

	: "Ir" (i));							\

}									\



#define ATOMIC64_OP_RETURN(op, asm_op)					\

static inline long atomic64_##op##_return(long i, atomic64_t *v)	\

{									\

	long result;							\

	unsigned long tmp;						\

									\

	asm volatile("// atomic64_" #op "_return\n"			\

"1:	ldxr	%0, %2\n"						\

"	" #asm_op "	%0, %0, %3\n"					\

"	stlxr	%w1, %0, %2\n"						\

"	cbnz	%w1, 1b"						\

	: "=&r" (result), "=&r" (tmp), "+Q" (v->counter)		\

	: "Ir" (i)							\

	: "memory");							\

									\

	smp_mb();							\

	return result;							\

}



static inline long atomic64_add_return(long i, atomic64_t *v)

{

	long result;

	unsigned long tmp;

#define ATOMIC64_OPS(op, asm_op)					\

	ATOMIC64_OP(op, asm_op)						\

	ATOMIC64_OP_RETURN(op, asm_op)



	asm volatile("// atomic64_add_return\n"

"1:	ldxr	%0, %2\n"

"	add	%0, %0, %3\n"

"	stlxr	%w1, %0, %2\n"

"	cbnz	%w1, 1b"

	: "=&r" (result), "=&r" (tmp), "+Q" (v->counter)

	: "Ir" (i)

	: "memory");

ATOMIC64_OPS(add, add)

ATOMIC64_OPS(sub, sub)



	smp_mb();

	return result;

}



static inline void atomic64_sub(u64 i, atomic64_t *v)

{

	long result;

	unsigned long tmp;



	asm volatile("// atomic64_sub\n"

"1:	ldxr	%0, %2\n"

"	sub	%0, %0, %3\n"

"	stxr	%w1, %0, %2\n"

"	cbnz	%w1, 1b"

	: "=&r" (result), "=&r" (tmp), "+Q" (v->counter)

	: "Ir" (i));

}



static inline long atomic64_sub_return(long i, atomic64_t *v)

{

	long result;

	unsigned long tmp;



	asm volatile("// atomic64_sub_return\n"

"1:	ldxr	%0, %2\n"

"	sub	%0, %0, %3\n"

"	stlxr	%w1, %0, %2\n"

"	cbnz	%w1, 1b"

	: "=&r" (result), "=&r" (tmp), "+Q" (v->counter)

	: "Ir" (i)

	: "memory");



	smp_mb();

	return result;

}

#undef ATOMIC64_OPS

#undef ATOMIC64_OP_RETURN

#undef ATOMIC64_OP



static inline long atomic64_cmpxchg(atomic64_t *ptr, long old, long new)

{

arch/avr32/include/asm/atomic.h

+63 −62

File changed.

Preview size limit exceeded, changes collapsed.

CRA Git | Maintained and supported by SUSTech CRA and CCSE