Commit 418ccbe3 authored by Nick Piggin's avatar Nick Piggin Committed by Thomas Gleixner
Browse files

x86: lock bitops 



I missed an obvious one!

x86 CPUs are defined not to reorder stores past earlier loads, so there is
no hardware memory barrier required to implement a release-consistent store
(all stores are, by definition).

So ditch the generic lock bitops, and implement optimised versions for x86,
which removes the mfence from __clear_bit_unlock (which is already a useful
primitive for SLUB).

Signed-off-by: default avatarNick Piggin <npiggin@suse.de>
Signed-off-by: default avatarIngo Molnar <mingo@elte.hu>
Signed-off-by: default avatarThomas Gleixner <tglx@linutronix.de>
parent ea580655

include/asm-x86/bitops_32.h

+42 −1
Original line number Diff line number Diff line
@@ -80,6 +80,20 @@ static inline void clear_bit(int nr, volatile unsigned long * addr) 
		:"Ir" (nr));

}



/*

 * clear_bit_unlock - Clears a bit in memory

 * @nr: Bit to clear

 * @addr: Address to start counting from

 *

 * clear_bit() is atomic and implies release semantics before the memory

 * operation. It can be used for an unlock.

 */

static inline void clear_bit_unlock(unsigned long nr, volatile unsigned long *addr)

{

	barrier();

	clear_bit(nr, addr);

}



static inline void __clear_bit(int nr, volatile unsigned long * addr)

{

	__asm__ __volatile__(
@@ -87,6 +101,25 @@ static inline void __clear_bit(int nr, volatile unsigned long * addr) 
		:"+m" (ADDR)

		:"Ir" (nr));

}



/*

 * __clear_bit_unlock - Clears a bit in memory

 * @nr: Bit to clear

 * @addr: Address to start counting from

 *

 * __clear_bit() is non-atomic and implies release semantics before the memory

 * operation. It can be used for an unlock if no other CPUs can concurrently

 * modify other bits in the word.

 *

 * No memory barrier is required here, because x86 cannot reorder stores past

 * older loads. Same principle as spin_unlock.

 */

static inline void __clear_bit_unlock(unsigned long nr, volatile unsigned long *addr)

{

	barrier();

	__clear_bit(nr, addr);

}



#define smp_mb__before_clear_bit()	barrier()

#define smp_mb__after_clear_bit()	barrier()
@@ -145,6 +178,15 @@ static inline int test_and_set_bit(int nr, volatile unsigned long * addr) 
	return oldbit;

}



/**

 * test_and_set_bit_lock - Set a bit and return its old value for lock

 * @nr: Bit to set

 * @addr: Address to count from

 *

 * This is the same as test_and_set_bit on x86

 */

#define test_and_set_bit_lock test_and_set_bit



/**

 * __test_and_set_bit - Set a bit and return its old value

 * @nr: Bit to set
@@ -406,7 +448,6 @@ static inline int fls(int x) 
}



#include <asm-generic/bitops/hweight.h>

#include <asm-generic/bitops/lock.h>



#endif /* __KERNEL__ */

include/asm-x86/bitops_64.h

+41 −1
Original line number Diff line number Diff line
@@ -72,6 +72,20 @@ static __inline__ void clear_bit(int nr, volatile void * addr) 
		:"dIr" (nr));

}



/*

 * clear_bit_unlock - Clears a bit in memory

 * @nr: Bit to clear

 * @addr: Address to start counting from

 *

 * clear_bit() is atomic and implies release semantics before the memory

 * operation. It can be used for an unlock.

 */

static inline void clear_bit_unlock(unsigned long nr, volatile unsigned long *addr)

{

	barrier();

	clear_bit(nr, addr);

}



static __inline__ void __clear_bit(int nr, volatile void * addr)

{

	__asm__ __volatile__(
@@ -80,6 +94,24 @@ static __inline__ void __clear_bit(int nr, volatile void * addr) 
		:"dIr" (nr));

}



/*

 * __clear_bit_unlock - Clears a bit in memory

 * @nr: Bit to clear

 * @addr: Address to start counting from

 *

 * __clear_bit() is non-atomic and implies release semantics before the memory

 * operation. It can be used for an unlock if no other CPUs can concurrently

 * modify other bits in the word.

 *

 * No memory barrier is required here, because x86 cannot reorder stores past

 * older loads. Same principle as spin_unlock.

 */

static inline void __clear_bit_unlock(unsigned long nr, volatile unsigned long *addr)

{

	barrier();

	__clear_bit(nr, addr);

}



#define smp_mb__before_clear_bit()	barrier()

#define smp_mb__after_clear_bit()	barrier()
@@ -136,6 +168,15 @@ static __inline__ int test_and_set_bit(int nr, volatile void * addr) 
	return oldbit;

}



/**

 * test_and_set_bit_lock - Set a bit and return its old value for lock

 * @nr: Bit to set

 * @addr: Address to count from

 *

 * This is the same as test_and_set_bit on x86

 */

#define test_and_set_bit_lock test_and_set_bit



/**

 * __test_and_set_bit - Set a bit and return its old value

 * @nr: Bit to set
@@ -412,7 +453,6 @@ static __inline__ int fls(int x) 
#define ARCH_HAS_FAST_MULTIPLIER 1



#include <asm-generic/bitops/hweight.h>

#include <asm-generic/bitops/lock.h>



#endif /* __KERNEL__ */

CRA Git | Maintained and supported by SUSTech CRA and CCSE