frv: Rewrite atomic implementation (b0d8003e) · Commits · 戴 / test

arch/frv/include/asm/atomic.h

+66 −53

Original line number	Original line	Diff line number	Diff line
	@@ -15,7 +15,6 @@
	#define _ASM_ATOMIC_H		#define _ASM_ATOMIC_H

	#include <linux/types.h>		#include <linux/types.h>
	#include <asm/spr-regs.h>
	#include <asm/cmpxchg.h>		#include <asm/cmpxchg.h>
	#include <asm/barrier.h>		#include <asm/barrier.h>

	@@ -23,6 +22,8 @@
	#error not SMP safe		#error not SMP safe
	#endif		#endif

			#include <asm/atomic_defs.h>

	/*		/*
	* Atomic operations that C can't guarantee us. Useful for		* Atomic operations that C can't guarantee us. Useful for
	* resource counting etc..		* resource counting etc..
	@@ -34,56 +35,26 @@
	#define atomic_read(v) ACCESS_ONCE((v)->counter)		#define atomic_read(v) ACCESS_ONCE((v)->counter)
	#define atomic_set(v, i) (((v)->counter) = (i))		#define atomic_set(v, i) (((v)->counter) = (i))

	#ifndef CONFIG_FRV_OUTOFLINE_ATOMIC_OPS		static inline int atomic_inc_return(atomic_t *v)
	static inline int atomic_add_return(int i, atomic_t *v)
	{		{
	unsigned long val;		return __atomic_add_return(1, &v->counter);
			}

	asm("0: \n"		static inline int atomic_dec_return(atomic_t *v)
	" orcc gr0,gr0,gr0,icc3 \n" /* set ICC3.Z */		{
	" ckeq icc3,cc7 \n"		return __atomic_sub_return(1, &v->counter);
	" ld.p %M0,%1 \n" /* LD.P/ORCR must be atomic */		}
	" orcr cc7,cc7,cc3 \n" /* set CC3 to true */
	" add%I2 %1,%2,%1 \n"
	" cst.p %1,%M0 ,cc3,#1 \n"
	" corcc gr29,gr29,gr0 ,cc3,#1 \n" /* clear ICC3.Z if store happens */
	" beq icc3,#0,0b \n"
	: "+U"(v->counter), "=&r"(val)
	: "NPr"(i)
	: "memory", "cc7", "cc3", "icc3"
	);

	return val;		static inline int atomic_add_return(int i, atomic_t *v)
			{
			return __atomic_add_return(i, &v->counter);
	}		}

	static inline int atomic_sub_return(int i, atomic_t *v)		static inline int atomic_sub_return(int i, atomic_t *v)
	{		{
	unsigned long val;		return __atomic_sub_return(i, &v->counter);

	asm("0: \n"
	" orcc gr0,gr0,gr0,icc3 \n" /* set ICC3.Z */
	" ckeq icc3,cc7 \n"
	" ld.p %M0,%1 \n" /* LD.P/ORCR must be atomic */
	" orcr cc7,cc7,cc3 \n" /* set CC3 to true */
	" sub%I2 %1,%2,%1 \n"
	" cst.p %1,%M0 ,cc3,#1 \n"
	" corcc gr29,gr29,gr0 ,cc3,#1 \n" /* clear ICC3.Z if store happens */
	" beq icc3,#0,0b \n"
	: "+U"(v->counter), "=&r"(val)
	: "NPr"(i)
	: "memory", "cc7", "cc3", "icc3"
	);

	return val;
	}		}

	#else

	extern int atomic_add_return(int i, atomic_t *v);
	extern int atomic_sub_return(int i, atomic_t *v);

	#endif

	static inline int atomic_add_negative(int i, atomic_t *v)		static inline int atomic_add_negative(int i, atomic_t *v)
	{		{
	return atomic_add_return(i, v) < 0;		return atomic_add_return(i, v) < 0;
	@@ -101,17 +72,14 @@ static inline void atomic_sub(int i, atomic_t *v)

	static inline void atomic_inc(atomic_t *v)		static inline void atomic_inc(atomic_t *v)
	{		{
	atomic_add_return(1, v);		atomic_inc_return(v);
	}		}

	static inline void atomic_dec(atomic_t *v)		static inline void atomic_dec(atomic_t *v)
	{		{
	atomic_sub_return(1, v);		atomic_dec_return(v);
	}		}

	#define atomic_dec_return(v) atomic_sub_return(1, (v))
	#define atomic_inc_return(v) atomic_add_return(1, (v))

	#define atomic_sub_and_test(i,v) (atomic_sub_return((i), (v)) == 0)		#define atomic_sub_and_test(i,v) (atomic_sub_return((i), (v)) == 0)
	#define atomic_dec_and_test(v) (atomic_sub_return(1, (v)) == 0)		#define atomic_dec_and_test(v) (atomic_sub_return(1, (v)) == 0)
	#define atomic_inc_and_test(v) (atomic_add_return(1, (v)) == 0)		#define atomic_inc_and_test(v) (atomic_add_return(1, (v)) == 0)
	@@ -120,18 +88,19 @@ static inline void atomic_dec(atomic_t *v)
	* 64-bit atomic ops		* 64-bit atomic ops
	*/		*/
	typedef struct {		typedef struct {
	volatile long long counter;		long long counter;
	} atomic64_t;		} atomic64_t;

	#define ATOMIC64_INIT(i) { (i) }		#define ATOMIC64_INIT(i) { (i) }

	static inline long long atomic64_read(atomic64_t *v)		static inline long long atomic64_read(const atomic64_t *v)
	{		{
	long long counter;		long long counter;

	asm("ldd%I1 %M1,%0"		asm("ldd%I1 %M1,%0"
	: "=e"(counter)		: "=e"(counter)
	: "m"(v->counter));		: "m"(v->counter));

	return counter;		return counter;
	}		}

	@@ -142,10 +111,25 @@ static inline void atomic64_set(atomic64_t *v, long long i)
	: "e"(i));		: "e"(i));
	}		}

	extern long long atomic64_inc_return(atomic64_t *v);		static inline long long atomic64_inc_return(atomic64_t *v)
	extern long long atomic64_dec_return(atomic64_t *v);		{
	extern long long atomic64_add_return(long long i, atomic64_t *v);		return __atomic64_add_return(1, &v->counter);
	extern long long atomic64_sub_return(long long i, atomic64_t *v);		}

			static inline long long atomic64_dec_return(atomic64_t *v)
			{
			return __atomic64_sub_return(1, &v->counter);
			}

			static inline long long atomic64_add_return(long long i, atomic64_t *v)
			{
			return __atomic64_add_return(i, &v->counter);
			}

			static inline long long atomic64_sub_return(long long i, atomic64_t *v)
			{
			return __atomic64_sub_return(i, &v->counter);
			}

	static inline long long atomic64_add_negative(long long i, atomic64_t *v)		static inline long long atomic64_add_negative(long long i, atomic64_t *v)
	{		{
	@@ -176,6 +160,7 @@ static inline void atomic64_dec(atomic64_t *v)
	#define atomic64_dec_and_test(v) (atomic64_dec_return((v)) == 0)		#define atomic64_dec_and_test(v) (atomic64_dec_return((v)) == 0)
	#define atomic64_inc_and_test(v) (atomic64_inc_return((v)) == 0)		#define atomic64_inc_and_test(v) (atomic64_inc_return((v)) == 0)


	#define atomic_cmpxchg(v, old, new) (cmpxchg(&(v)->counter, old, new))		#define atomic_cmpxchg(v, old, new) (cmpxchg(&(v)->counter, old, new))
	#define atomic_xchg(v, new) (xchg(&(v)->counter, new))		#define atomic_xchg(v, new) (xchg(&(v)->counter, new))
	#define atomic64_cmpxchg(v, old, new) (__cmpxchg_64(old, new, &(v)->counter))		#define atomic64_cmpxchg(v, old, new) (__cmpxchg_64(old, new, &(v)->counter))
	@@ -196,5 +181,33 @@ static __inline__ int __atomic_add_unless(atomic_t *v, int a, int u)
	return c;		return c;
	}		}

			#define ATOMIC_OP(op) \
			static inline void atomic_##op(int i, atomic_t *v) \
			{ \
			(void)__atomic32_fetch_##op(i, &v->counter); \
			} \
			\
			static inline void atomic64_##op(long long i, atomic64_t *v) \
			{ \
			(void)__atomic64_fetch_##op(i, &v->counter); \
			}

			#define CONFIG_ARCH_HAS_ATOMIC_OR

			ATOMIC_OP(or)
			ATOMIC_OP(and)
			ATOMIC_OP(xor)

			#undef ATOMIC_OP

			static inline __deprecated void atomic_clear_mask(unsigned int mask, atomic_t *v)
			{
			atomic_and(~mask, v);
			}

			static inline __deprecated void atomic_set_mask(unsigned int mask, atomic_t *v)
			{
			atomic_or(mask, v);
			}

	#endif /* _ASM_ATOMIC_H */		#endif /* _ASM_ATOMIC_H */

arch/frv/include/asm/atomic_defs.h

0 → 100644

+172 −0

Original line number	Original line	Diff line number	Diff line

			#include <asm/spr-regs.h>

			#ifdef __ATOMIC_LIB__

			#ifdef CONFIG_FRV_OUTOFLINE_ATOMIC_OPS

			#define ATOMIC_QUALS
			#define ATOMIC_EXPORT(x) EXPORT_SYMBOL(x)

			#else /* !OUTOFLINE && LIB */

			#define ATOMIC_OP_RETURN(op)
			#define ATOMIC_FETCH_OP(op)

			#endif /* OUTOFLINE */

			#else /* !__ATOMIC_LIB__ */

			#define ATOMIC_EXPORT(x)

			#ifdef CONFIG_FRV_OUTOFLINE_ATOMIC_OPS

			#define ATOMIC_OP_RETURN(op) \
			extern int __atomic_##op##_return(int i, int *v); \
			extern long long __atomic64_##op##_return(long long i, long long *v);

			#define ATOMIC_FETCH_OP(op) \
			extern int __atomic32_fetch_##op(int i, int *v); \
			extern long long __atomic64_fetch_##op(long long i, long long *v);

			#else /* !OUTOFLINE && !LIB */

			#define ATOMIC_QUALS static inline

			#endif /* OUTOFLINE */
			#endif /* __ATOMIC_LIB__ */


			/*
			* Note on the 64 bit inline asm variants...
			*
			* CSTD is a conditional instruction and needs a constrained memory reference.
			* Normally 'U' provides the correct constraints for conditional instructions
			* and this is used for the 32 bit version, however 'U' does not appear to work
			* for 64 bit values (gcc-4.9)
			*
			* The exact constraint is that conditional instructions cannot deal with an
			* immediate displacement in the memory reference, so what we do is we read the
			* address through a volatile cast into a local variable in order to insure we
			* _have_ to compute the correct address without displacement. This allows us
			* to use the regular 'm' for the memory address.
			*
			* Furthermore, the %Ln operand, which prints the low word register (r+1),
			* really only works for registers, this means we cannot allow immediate values
			* for the 64 bit versions -- like we do for the 32 bit ones.
			*
			*/

			#ifndef ATOMIC_OP_RETURN
			#define ATOMIC_OP_RETURN(op) \
			ATOMIC_QUALS int __atomic_##op##_return(int i, int *v) \
			{ \
			int val; \
			\
			asm volatile( \
			"0: \n" \
			" orcc gr0,gr0,gr0,icc3 \n" \
			" ckeq icc3,cc7 \n" \
			" ld.p %M0,%1 \n" \
			" orcr cc7,cc7,cc3 \n" \
			" "#op"%I2 %1,%2,%1 \n" \
			" cst.p %1,%M0 ,cc3,#1 \n" \
			" corcc gr29,gr29,gr0 ,cc3,#1 \n" \
			" beq icc3,#0,0b \n" \
			: "+U"(*v), "=&r"(val) \
			: "NPr"(i) \
			: "memory", "cc7", "cc3", "icc3" \
			); \
			\
			return val; \
			} \
			ATOMIC_EXPORT(__atomic_##op##_return); \
			\
			ATOMIC_QUALS long long __atomic64_##op##_return(long long i, long long *v) \
			{ \
			long long *__v = READ_ONCE(v); \
			long long val; \
			\
			asm volatile( \
			"0: \n" \
			" orcc gr0,gr0,gr0,icc3 \n" \
			" ckeq icc3,cc7 \n" \
			" ldd.p %M0,%1 \n" \
			" orcr cc7,cc7,cc3 \n" \
			" "#op"cc %L1,%L2,%L1,icc0 \n" \
			" "#op"x %1,%2,%1,icc0 \n" \
			" cstd.p %1,%M0 ,cc3,#1 \n" \
			" corcc gr29,gr29,gr0 ,cc3,#1 \n" \
			" beq icc3,#0,0b \n" \
			: "+m"(*__v), "=&e"(val) \
			: "e"(i) \
			: "memory", "cc7", "cc3", "icc0", "icc3" \
			); \
			\
			return val; \
			} \
			ATOMIC_EXPORT(__atomic64_##op##_return);
			#endif

			#ifndef ATOMIC_FETCH_OP
			#define ATOMIC_FETCH_OP(op) \
			ATOMIC_QUALS int __atomic32_fetch_##op(int i, int *v) \
			{ \
			int old, tmp; \
			\
			asm volatile( \
			"0: \n" \
			" orcc gr0,gr0,gr0,icc3 \n" \
			" ckeq icc3,cc7 \n" \
			" ld.p %M0,%1 \n" \
			" orcr cc7,cc7,cc3 \n" \
			" "#op"%I3 %1,%3,%2 \n" \
			" cst.p %2,%M0 ,cc3,#1 \n" \
			" corcc gr29,gr29,gr0 ,cc3,#1 \n" \
			" beq icc3,#0,0b \n" \
			: "+U"(*v), "=&r"(old), "=r"(tmp) \
			: "NPr"(i) \
			: "memory", "cc7", "cc3", "icc3" \
			); \
			\
			return old; \
			} \
			ATOMIC_EXPORT(__atomic32_fetch_##op); \
			\
			ATOMIC_QUALS long long __atomic64_fetch_##op(long long i, long long *v) \
			{ \
			long long *__v = READ_ONCE(v); \
			long long old, tmp; \
			\
			asm volatile( \
			"0: \n" \
			" orcc gr0,gr0,gr0,icc3 \n" \
			" ckeq icc3,cc7 \n" \
			" ldd.p %M0,%1 \n" \
			" orcr cc7,cc7,cc3 \n" \
			" "#op" %L1,%L3,%L2 \n" \
			" "#op" %1,%3,%2 \n" \
			" cstd.p %2,%M0 ,cc3,#1 \n" \
			" corcc gr29,gr29,gr0 ,cc3,#1 \n" \
			" beq icc3,#0,0b \n" \
			: "+m"(*__v), "=&e"(old), "=e"(tmp) \
			: "e"(i) \
			: "memory", "cc7", "cc3", "icc3" \
			); \
			\
			return old; \
			} \
			ATOMIC_EXPORT(__atomic64_fetch_##op);
			#endif

			ATOMIC_FETCH_OP(or)
			ATOMIC_FETCH_OP(and)
			ATOMIC_FETCH_OP(xor)

			ATOMIC_OP_RETURN(add)
			ATOMIC_OP_RETURN(sub)

			#undef ATOMIC_FETCH_OP
			#undef ATOMIC_OP_RETURN
			#undef ATOMIC_QUALS
			#undef ATOMIC_EXPORT

arch/frv/include/asm/bitops.h

+10 −89

Original line number	Original line	Diff line number	Diff line
	@@ -25,109 +25,30 @@

	#include <asm-generic/bitops/ffz.h>		#include <asm-generic/bitops/ffz.h>

	#ifndef CONFIG_FRV_OUTOFLINE_ATOMIC_OPS		#include <asm/atomic.h>
	static inline
	unsigned long atomic_test_and_ANDNOT_mask(unsigned long mask, volatile unsigned long *v)
	{
	unsigned long old, tmp;

	asm volatile(
	"0: \n"
	" orcc gr0,gr0,gr0,icc3 \n" /* set ICC3.Z */
	" ckeq icc3,cc7 \n"
	" ld.p %M0,%1 \n" /* LD.P/ORCR are atomic */
	" orcr cc7,cc7,cc3 \n" /* set CC3 to true */
	" and%I3 %1,%3,%2 \n"
	" cst.p %2,%M0 ,cc3,#1 \n" /* if store happens... */
	" corcc gr29,gr29,gr0 ,cc3,#1 \n" /* ... clear ICC3.Z */
	" beq icc3,#0,0b \n"
	: "+U"(*v), "=&r"(old), "=r"(tmp)
	: "NPr"(~mask)
	: "memory", "cc7", "cc3", "icc3"
	);

	return old;
	}

	static inline
	unsigned long atomic_test_and_OR_mask(unsigned long mask, volatile unsigned long *v)
	{
	unsigned long old, tmp;

	asm volatile(
	"0: \n"
	" orcc gr0,gr0,gr0,icc3 \n" /* set ICC3.Z */
	" ckeq icc3,cc7 \n"
	" ld.p %M0,%1 \n" /* LD.P/ORCR are atomic */
	" orcr cc7,cc7,cc3 \n" /* set CC3 to true */
	" or%I3 %1,%3,%2 \n"
	" cst.p %2,%M0 ,cc3,#1 \n" /* if store happens... */
	" corcc gr29,gr29,gr0 ,cc3,#1 \n" /* ... clear ICC3.Z */
	" beq icc3,#0,0b \n"
	: "+U"(*v), "=&r"(old), "=r"(tmp)
	: "NPr"(mask)
	: "memory", "cc7", "cc3", "icc3"
	);

	return old;
	}

	static inline
	unsigned long atomic_test_and_XOR_mask(unsigned long mask, volatile unsigned long *v)
	{
	unsigned long old, tmp;

	asm volatile(
	"0: \n"
	" orcc gr0,gr0,gr0,icc3 \n" /* set ICC3.Z */
	" ckeq icc3,cc7 \n"
	" ld.p %M0,%1 \n" /* LD.P/ORCR are atomic */
	" orcr cc7,cc7,cc3 \n" /* set CC3 to true */
	" xor%I3 %1,%3,%2 \n"
	" cst.p %2,%M0 ,cc3,#1 \n" /* if store happens... */
	" corcc gr29,gr29,gr0 ,cc3,#1 \n" /* ... clear ICC3.Z */
	" beq icc3,#0,0b \n"
	: "+U"(*v), "=&r"(old), "=r"(tmp)
	: "NPr"(mask)
	: "memory", "cc7", "cc3", "icc3"
	);

	return old;
	}

	#else

	extern unsigned long atomic_test_and_ANDNOT_mask(unsigned long mask, volatile unsigned long *v);
	extern unsigned long atomic_test_and_OR_mask(unsigned long mask, volatile unsigned long *v);
	extern unsigned long atomic_test_and_XOR_mask(unsigned long mask, volatile unsigned long *v);

	#endif

	#define atomic_clear_mask(mask, v) atomic_test_and_ANDNOT_mask((mask), (v))
	#define atomic_set_mask(mask, v) atomic_test_and_OR_mask((mask), (v))

	static inline int test_and_clear_bit(unsigned long nr, volatile void *addr)		static inline int test_and_clear_bit(unsigned long nr, volatile void *addr)
	{		{
	volatile unsigned long *ptr = addr;		unsigned int ptr = (void )addr;
	unsigned long mask = 1UL << (nr & 31);		unsigned int mask = 1UL << (nr & 31);
	ptr += nr >> 5;		ptr += nr >> 5;
	return (atomic_test_and_ANDNOT_mask(mask, ptr) & mask) != 0;		return (__atomic32_fetch_and(~mask, ptr) & mask) != 0;
	}		}

	static inline int test_and_set_bit(unsigned long nr, volatile void *addr)		static inline int test_and_set_bit(unsigned long nr, volatile void *addr)
	{		{
	volatile unsigned long *ptr = addr;		unsigned int ptr = (void )addr;
	unsigned long mask = 1UL << (nr & 31);		unsigned int mask = 1UL << (nr & 31);
	ptr += nr >> 5;		ptr += nr >> 5;
	return (atomic_test_and_OR_mask(mask, ptr) & mask) != 0;		return (__atomic32_fetch_or(mask, ptr) & mask) != 0;
	}		}

	static inline int test_and_change_bit(unsigned long nr, volatile void *addr)		static inline int test_and_change_bit(unsigned long nr, volatile void *addr)
	{		{
	volatile unsigned long *ptr = addr;		unsigned int ptr = (void )addr;
	unsigned long mask = 1UL << (nr & 31);		unsigned int mask = 1UL << (nr & 31);
	ptr += nr >> 5;		ptr += nr >> 5;
	return (atomic_test_and_XOR_mask(mask, ptr) & mask) != 0;		return (__atomic32_fetch_xor(mask, ptr) & mask) != 0;
	}		}

	static inline void clear_bit(unsigned long nr, volatile void *addr)		static inline void clear_bit(unsigned long nr, volatile void *addr)

arch/frv/kernel/dma.c

+3 −3

Original line number	Original line	Diff line number	Diff line
	@@ -109,13 +109,13 @@ static struct frv_dma_channel frv_dma_channels[FRV_DMA_NCHANS] = {

	static DEFINE_RWLOCK(frv_dma_channels_lock);		static DEFINE_RWLOCK(frv_dma_channels_lock);

	unsigned long frv_dma_inprogress;		unsigned int frv_dma_inprogress;

	#define frv_clear_dma_inprogress(channel) \		#define frv_clear_dma_inprogress(channel) \
	atomic_clear_mask(1 << (channel), &frv_dma_inprogress);		(void)__atomic32_fetch_and(~(1 << (channel)), &frv_dma_inprogress);

	#define frv_set_dma_inprogress(channel) \		#define frv_set_dma_inprogress(channel) \
	atomic_set_mask(1 << (channel), &frv_dma_inprogress);		(void)__atomic32_fetch_or(1 << (channel), &frv_dma_inprogress);

	/*****************************************************************************/		/*****************************************************************************/
	/*		/*

arch/frv/kernel/frv_ksyms.c

+0 −5

Original line number	Original line	Diff line number	Diff line
	@@ -58,11 +58,6 @@ EXPORT_SYMBOL(__outsl_ns);
	EXPORT_SYMBOL(__insl_ns);		EXPORT_SYMBOL(__insl_ns);

	#ifdef CONFIG_FRV_OUTOFLINE_ATOMIC_OPS		#ifdef CONFIG_FRV_OUTOFLINE_ATOMIC_OPS
	EXPORT_SYMBOL(atomic_test_and_ANDNOT_mask);
	EXPORT_SYMBOL(atomic_test_and_OR_mask);
	EXPORT_SYMBOL(atomic_test_and_XOR_mask);
	EXPORT_SYMBOL(atomic_add_return);
	EXPORT_SYMBOL(atomic_sub_return);
	EXPORT_SYMBOL(__xchg_32);		EXPORT_SYMBOL(__xchg_32);
	EXPORT_SYMBOL(__cmpxchg_32);		EXPORT_SYMBOL(__cmpxchg_32);
	#endif		#endif

Admin message