Unverified Commit 4d1dbfe6 authored by Paul Burton's avatar Paul Burton
Browse files

MIPS: atomic: Emit Loongson3 sync workarounds within asm 



Generate the sync instructions required to workaround Loongson3 LL/SC
errata within inline asm blocks, which feels a little safer than doing
it from C where strictly speaking the compiler would be well within its
rights to insert a memory access between the separate asm statements we
previously had, containing sync & ll instructions respectively.

Signed-off-by: default avatarPaul Burton <paul.burton@mips.com>
Cc: linux-mips@vger.kernel.org
Cc: Huacai Chen <chenhc@lemote.com>
Cc: Jiaxun Yang <jiaxun.yang@flygoat.com>
Cc: linux-kernel@vger.kernel.org
parent a38ee6bb

arch/mips/include/asm/atomic.h

+14 −6
Original line number Diff line number Diff line
@@ -21,6 +21,7 @@ 
#include <asm/cpu-features.h>

#include <asm/cmpxchg.h>

#include <asm/llsc.h>

#include <asm/sync.h>

#include <asm/war.h>



#define ATOMIC_INIT(i)	  { (i) }
@@ -56,10 +57,10 @@ static __inline__ void pfx##_##op(type i, pfx##_t * v) \ 
		return;							\

	}								\

									\

	loongson_llsc_mb();						\

	__asm__ __volatile__(						\

	"	.set	push					\n"	\

	"	.set	" MIPS_ISA_LEVEL "			\n"	\

	"	" __SYNC(full, loongson3_war) "			\n"	\

	"1:	" #ll "	%0, %1		# " #pfx "_" #op "	\n"	\

	"	" #asm_op " %0, %2				\n"	\

	"	" #sc "	%0, %1					\n"	\
@@ -85,10 +86,10 @@ static __inline__ type pfx##_##op##_return_relaxed(type i, pfx##_t * v) \ 
		return result;						\

	}								\

									\

	loongson_llsc_mb();						\

	__asm__ __volatile__(						\

	"	.set	push					\n"	\

	"	.set	" MIPS_ISA_LEVEL "			\n"	\

	"	" __SYNC(full, loongson3_war) "			\n"	\

	"1:	" #ll "	%1, %2		# " #pfx "_" #op "_return\n"	\

	"	" #asm_op " %0, %1, %3				\n"	\

	"	" #sc "	%0, %2					\n"	\
@@ -117,10 +118,10 @@ static __inline__ type pfx##_fetch_##op##_relaxed(type i, pfx##_t * v) \ 
		return result;						\

	}								\

									\

	loongson_llsc_mb();						\

	__asm__ __volatile__(						\

	"	.set	push					\n"	\

	"	.set	" MIPS_ISA_LEVEL "			\n"	\

	"	" __SYNC(full, loongson3_war) "			\n"	\

	"1:	" #ll "	%1, %2		# " #pfx "_fetch_" #op "\n"	\

	"	" #asm_op " %0, %1, %3				\n"	\

	"	" #sc "	%0, %2					\n"	\
@@ -200,10 +201,10 @@ static __inline__ int atomic_sub_if_positive(int i, atomic_t * v) 
	if (kernel_uses_llsc) {

		int temp;



		loongson_llsc_mb();

		__asm__ __volatile__(

		"	.set	push					\n"

		"	.set	"MIPS_ISA_LEVEL"			\n"

		"	" __SYNC(full, loongson3_war) "			\n"

		"1:	ll	%1, %2		# atomic_sub_if_positive\n"

		"	.set	pop					\n"

		"	subu	%0, %1, %3				\n"
@@ -213,7 +214,7 @@ static __inline__ int atomic_sub_if_positive(int i, atomic_t * v) 
		"	.set	"MIPS_ISA_LEVEL"			\n"

		"	sc	%1, %2					\n"

		"\t" __SC_BEQZ "%1, 1b					\n"

		"2:							\n"

		"2:	" __SYNC(full, loongson3_war) "			\n"

		"	.set	pop					\n"

		: "=&r" (result), "=&r" (temp),

		  "+" GCC_OFF_SMALL_ASM() (v->counter)
@@ -229,6 +230,13 @@ static __inline__ int atomic_sub_if_positive(int i, atomic_t * v) 
		raw_local_irq_restore(flags);

	}



	/*

	 * In the Loongson3 workaround case we already have a completion

	 * barrier at 2: above, which is needed due to the bltz that can branch

	 * to code outside of the LL/SC loop. As such, we don't need to emit

	 * another barrier here.

	 */

	if (!__SYNC_loongson3_war)

		smp_llsc_mb();



	return result;

CRA Git | Maintained and supported by SUSTech CRA and CCSE