Commit 0ccc8b7a authored by Heiko Carstens's avatar Heiko Carstens Committed by Martin Schwidefsky
Browse files

s390/bitops,atomic: add missing memory barriers 



When reworking the bitops and atomic ops I missed that those instructions
that got atomic behaviour only perform a "specific-operand-serialization"
instead of a full "serialization".
The compare-and-swap instruction used before performs a full serialization
before and after the instruction is executed, which means it has full
memory barrier semantics.
In order to give the new bitops and atomic ops functions also full memory
barrier semantics add a "bcr 14,0" before and after each of those new
instructions which performs full serialization as well.

This restores memory barrier semantics for bitops and atomic ops functions
which return values, like e.g. atomic_add_return(), but not for functions
which do not return a value, like e.g. atomic_add().
This is consistent to other architectures and what common code requires.

Cc: stable@vger.kernel.org # v3.13+
Signed-off-by: default avatarHeiko Carstens <heiko.carstens@de.ibm.com>
Signed-off-by: default avatarMartin Schwidefsky <schwidefsky@de.ibm.com>
parent ce1ce2f3

arch/s390/include/asm/atomic.h

+42 −28
Original line number Diff line number Diff line
@@ -15,23 +15,29 @@ 


#include <linux/compiler.h>

#include <linux/types.h>

#include <asm/barrier.h>

#include <asm/cmpxchg.h>



#define ATOMIC_INIT(i)  { (i) }



#define __ATOMIC_NO_BARRIER	"\n"



#ifdef CONFIG_HAVE_MARCH_Z196_FEATURES



#define __ATOMIC_OR	"lao"

#define __ATOMIC_AND	"lan"

#define __ATOMIC_ADD	"laa"

#define __ATOMIC_BARRIER "bcr	14,0\n"



#define __ATOMIC_LOOP(ptr, op_val, op_string)				\

#define __ATOMIC_LOOP(ptr, op_val, op_string, __barrier)		\

({									\

	int old_val;							\

									\

	typecheck(atomic_t *, ptr);					\

	asm volatile(							\

		__barrier						\

		op_string "	%0,%2,%1\n"				\

		__barrier						\

		: "=d" (old_val), "+Q" ((ptr)->counter)			\

		: "d" (op_val)						\

		: "cc", "memory");					\
@@ -43,8 +49,9 @@ 
#define __ATOMIC_OR	"or"

#define __ATOMIC_AND	"nr"

#define __ATOMIC_ADD	"ar"

#define __ATOMIC_BARRIER "\n"



#define __ATOMIC_LOOP(ptr, op_val, op_string)				\

#define __ATOMIC_LOOP(ptr, op_val, op_string, __barrier)		\

({									\

	int old_val, new_val;						\

									\
@@ -82,7 +89,7 @@ static inline void atomic_set(atomic_t *v, int i) 


static inline int atomic_add_return(int i, atomic_t *v)

{

	return __ATOMIC_LOOP(v, i, __ATOMIC_ADD) + i;

	return __ATOMIC_LOOP(v, i, __ATOMIC_ADD, __ATOMIC_BARRIER) + i;

}



static inline void atomic_add(int i, atomic_t *v)
@@ -94,12 +101,10 @@ static inline void atomic_add(int i, atomic_t *v) 
			: "+Q" (v->counter)

			: "i" (i)

			: "cc", "memory");

	} else {

		atomic_add_return(i, v);

		return;

	}

#else

	atomic_add_return(i, v);

#endif

	__ATOMIC_LOOP(v, i, __ATOMIC_ADD, __ATOMIC_NO_BARRIER);

}



#define atomic_add_negative(_i, _v)	(atomic_add_return(_i, _v) < 0)
@@ -115,12 +120,12 @@ static inline void atomic_add(int i, atomic_t *v) 


static inline void atomic_clear_mask(unsigned int mask, atomic_t *v)

{

	__ATOMIC_LOOP(v, ~mask, __ATOMIC_AND);

	__ATOMIC_LOOP(v, ~mask, __ATOMIC_AND, __ATOMIC_NO_BARRIER);

}



static inline void atomic_set_mask(unsigned int mask, atomic_t *v)

{

	__ATOMIC_LOOP(v, mask, __ATOMIC_OR);

	__ATOMIC_LOOP(v, mask, __ATOMIC_OR, __ATOMIC_NO_BARRIER);

}



#define atomic_xchg(v, new) (xchg(&((v)->counter), new))
@@ -157,19 +162,24 @@ static inline int __atomic_add_unless(atomic_t *v, int a, int u) 


#ifdef CONFIG_64BIT



#define __ATOMIC64_NO_BARRIER	"\n"



#ifdef CONFIG_HAVE_MARCH_Z196_FEATURES



#define __ATOMIC64_OR	"laog"

#define __ATOMIC64_AND	"lang"

#define __ATOMIC64_ADD	"laag"

#define __ATOMIC64_BARRIER "bcr	14,0\n"



#define __ATOMIC64_LOOP(ptr, op_val, op_string)				\

#define __ATOMIC64_LOOP(ptr, op_val, op_string, __barrier)		\

({									\

	long long old_val;						\

									\

	typecheck(atomic64_t *, ptr);					\

	asm volatile(							\

		__barrier						\

		op_string "	%0,%2,%1\n"				\

		__barrier						\

		: "=d" (old_val), "+Q" ((ptr)->counter)			\

		: "d" (op_val)						\

		: "cc", "memory");					\
@@ -181,8 +191,9 @@ static inline int __atomic_add_unless(atomic_t *v, int a, int u) 
#define __ATOMIC64_OR	"ogr"

#define __ATOMIC64_AND	"ngr"

#define __ATOMIC64_ADD	"agr"

#define __ATOMIC64_BARRIER "\n"



#define __ATOMIC64_LOOP(ptr, op_val, op_string)				\

#define __ATOMIC64_LOOP(ptr, op_val, op_string, __barrier)		\

({									\

	long long old_val, new_val;					\

									\
@@ -220,17 +231,32 @@ static inline void atomic64_set(atomic64_t *v, long long i) 


static inline long long atomic64_add_return(long long i, atomic64_t *v)

{

	return __ATOMIC64_LOOP(v, i, __ATOMIC64_ADD) + i;

	return __ATOMIC64_LOOP(v, i, __ATOMIC64_ADD, __ATOMIC64_BARRIER) + i;

}



static inline void atomic64_add(long long i, atomic64_t *v)

{

#ifdef CONFIG_HAVE_MARCH_Z196_FEATURES

	if (__builtin_constant_p(i) && (i > -129) && (i < 128)) {

		asm volatile(

			"agsi	%0,%1\n"

			: "+Q" (v->counter)

			: "i" (i)

			: "cc", "memory");

		return;

	}

#endif

	__ATOMIC64_LOOP(v, i, __ATOMIC64_ADD, __ATOMIC64_NO_BARRIER);

}



static inline void atomic64_clear_mask(unsigned long mask, atomic64_t *v)

{

	__ATOMIC64_LOOP(v, ~mask, __ATOMIC64_AND);

	__ATOMIC64_LOOP(v, ~mask, __ATOMIC64_AND, __ATOMIC64_NO_BARRIER);

}



static inline void atomic64_set_mask(unsigned long mask, atomic64_t *v)

{

	__ATOMIC64_LOOP(v, mask, __ATOMIC64_OR);

	__ATOMIC64_LOOP(v, mask, __ATOMIC64_OR, __ATOMIC64_NO_BARRIER);

}



#define atomic64_xchg(v, new) (xchg(&((v)->counter), new))
@@ -334,24 +360,12 @@ static inline void atomic64_clear_mask(unsigned long long mask, atomic64_t *v) 
	} while (atomic64_cmpxchg(v, old, new) != old);

}



#endif /* CONFIG_64BIT */



static inline void atomic64_add(long long i, atomic64_t *v)

{

#ifdef CONFIG_HAVE_MARCH_Z196_FEATURES

	if (__builtin_constant_p(i) && (i > -129) && (i < 128)) {

		asm volatile(

			"agsi	%0,%1\n"

			: "+Q" (v->counter)

			: "i" (i)

			: "cc", "memory");

	} else {

	atomic64_add_return(i, v);

}

#else

	atomic64_add_return(i, v);

#endif

}



#endif /* CONFIG_64BIT */



static inline int atomic64_add_unless(atomic64_t *v, long long i, long long u)

{

arch/s390/include/asm/bitops.h

+23 −18
Original line number Diff line number Diff line
@@ -47,14 +47,18 @@ 


#include <linux/typecheck.h>

#include <linux/compiler.h>

#include <asm/barrier.h>



#define __BITOPS_NO_BARRIER	"\n"



#ifndef CONFIG_64BIT



#define __BITOPS_OR		"or"

#define __BITOPS_AND		"nr"

#define __BITOPS_XOR		"xr"

#define __BITOPS_BARRIER	"\n"



#define __BITOPS_LOOP(__addr, __val, __op_string)		\

#define __BITOPS_LOOP(__addr, __val, __op_string, __barrier)	\

({								\

	unsigned long __old, __new;				\

								\
@@ -67,7 +71,7 @@ 
		"	jl	0b"				\

		: "=&d" (__old), "=&d" (__new), "+Q" (*(__addr))\

		: "d" (__val)					\

		: "cc");					\

		: "cc", "memory");				\

	__old;							\

})
@@ -78,17 +82,20 @@ 
#define __BITOPS_OR		"laog"

#define __BITOPS_AND		"lang"

#define __BITOPS_XOR		"laxg"

#define __BITOPS_BARRIER	"bcr	14,0\n"



#define __BITOPS_LOOP(__addr, __val, __op_string)		\

#define __BITOPS_LOOP(__addr, __val, __op_string, __barrier)	\

({								\

	unsigned long __old;					\

								\

	typecheck(unsigned long *, (__addr));			\

	asm volatile(						\

		__barrier					\

		__op_string "	%0,%2,%1\n"			\

		__barrier					\

		: "=d" (__old),	"+Q" (*(__addr))		\

		: "d" (__val)					\

		: "cc");					\

		: "cc", "memory");				\

	__old;							\

})
@@ -97,8 +104,9 @@ 
#define __BITOPS_OR		"ogr"

#define __BITOPS_AND		"ngr"

#define __BITOPS_XOR		"xgr"

#define __BITOPS_BARRIER	"\n"



#define __BITOPS_LOOP(__addr, __val, __op_string)		\

#define __BITOPS_LOOP(__addr, __val, __op_string, __barrier)	\

({								\

	unsigned long __old, __new;				\

								\
@@ -111,7 +119,7 @@ 
		"	jl	0b"				\

		: "=&d" (__old), "=&d" (__new), "+Q" (*(__addr))\

		: "d" (__val)					\

		: "cc");					\

		: "cc", "memory");				\

	__old;							\

})
@@ -149,12 +157,12 @@ static inline void set_bit(unsigned long nr, volatile unsigned long *ptr) 
			"oi	%0,%b1\n"

			: "+Q" (*caddr)

			: "i" (1 << (nr & 7))

			: "cc");

			: "cc", "memory");

		return;

	}

#endif

	mask = 1UL << (nr & (BITS_PER_LONG - 1));

	__BITOPS_LOOP(addr, mask, __BITOPS_OR);

	__BITOPS_LOOP(addr, mask, __BITOPS_OR, __BITOPS_NO_BARRIER);

}



static inline void clear_bit(unsigned long nr, volatile unsigned long *ptr)
@@ -170,12 +178,12 @@ static inline void clear_bit(unsigned long nr, volatile unsigned long *ptr) 
			"ni	%0,%b1\n"

			: "+Q" (*caddr)

			: "i" (~(1 << (nr & 7)))

			: "cc");

			: "cc", "memory");

		return;

	}

#endif

	mask = ~(1UL << (nr & (BITS_PER_LONG - 1)));

	__BITOPS_LOOP(addr, mask, __BITOPS_AND);

	__BITOPS_LOOP(addr, mask, __BITOPS_AND, __BITOPS_NO_BARRIER);

}



static inline void change_bit(unsigned long nr, volatile unsigned long *ptr)
@@ -191,12 +199,12 @@ static inline void change_bit(unsigned long nr, volatile unsigned long *ptr) 
			"xi	%0,%b1\n"

			: "+Q" (*caddr)

			: "i" (1 << (nr & 7))

			: "cc");

			: "cc", "memory");

		return;

	}

#endif

	mask = 1UL << (nr & (BITS_PER_LONG - 1));

	__BITOPS_LOOP(addr, mask, __BITOPS_XOR);

	__BITOPS_LOOP(addr, mask, __BITOPS_XOR, __BITOPS_NO_BARRIER);

}



static inline int
@@ -206,8 +214,7 @@ test_and_set_bit(unsigned long nr, volatile unsigned long *ptr) 
	unsigned long old, mask;



	mask = 1UL << (nr & (BITS_PER_LONG - 1));

	old = __BITOPS_LOOP(addr, mask, __BITOPS_OR);

	barrier();

	old = __BITOPS_LOOP(addr, mask, __BITOPS_OR, __BITOPS_BARRIER);

	return (old & mask) != 0;

}
@@ -218,8 +225,7 @@ test_and_clear_bit(unsigned long nr, volatile unsigned long *ptr) 
	unsigned long old, mask;



	mask = ~(1UL << (nr & (BITS_PER_LONG - 1)));

	old = __BITOPS_LOOP(addr, mask, __BITOPS_AND);

	barrier();

	old = __BITOPS_LOOP(addr, mask, __BITOPS_AND, __BITOPS_BARRIER);

	return (old & ~mask) != 0;

}
@@ -230,8 +236,7 @@ test_and_change_bit(unsigned long nr, volatile unsigned long *ptr) 
	unsigned long old, mask;



	mask = 1UL << (nr & (BITS_PER_LONG - 1));

	old = __BITOPS_LOOP(addr, mask, __BITOPS_XOR);

	barrier();

	old = __BITOPS_LOOP(addr, mask, __BITOPS_XOR, __BITOPS_BARRIER);

	return (old & mask) != 0;

}

CRA Git | Maintained and supported by SUSTech CRA and CCSE