Commit 6323f0cc authored by Russell King's avatar Russell King
Browse files

ARM: bitops: switch set/clear/change bitops to use ldrex/strex 



Switch the set/clear/change bitops to use the word-based exclusive
operations, which are only present in a wider range of ARM architectures
than the byte-based exclusive operations.

Tested record:
- Nicolas Pitre: ext3,rw,le
- Sourav Poddar: nfs,le
- Will Deacon: ext3,rw,le
- Tony Lindgren: ext3+nfs,le

Reviewed-by: default avatarNicolas Pitre <nicolas.pitre@linaro.org>
Tested-by: default avatarSourav Poddar <sourav.poddar@ti.com>
Tested-by: default avatarWill Deacon <will.deacon@arm.com>
Tested-by: default avatarTony Lindgren <tony@atomide.com>
Signed-off-by: default avatarRussell King <rmk+kernel@arm.linux.org.uk>
parent a16ede35

arch/arm/include/asm/bitops.h

+22 −38
Original line number Diff line number Diff line
@@ -148,15 +148,19 @@ ____atomic_test_and_change_bit(unsigned int bit, volatile unsigned long *p) 
 * Note that bit 0 is defined to be 32-bit word bit 0, not byte 0 bit 0.

 */



/*

 * Native endian assembly bitops.  nr = 0 -> word 0 bit 0.

 */

extern void _set_bit(int nr, volatile unsigned long * p);

extern void _clear_bit(int nr, volatile unsigned long * p);

extern void _change_bit(int nr, volatile unsigned long * p);

extern int _test_and_set_bit(int nr, volatile unsigned long * p);

extern int _test_and_clear_bit(int nr, volatile unsigned long * p);

extern int _test_and_change_bit(int nr, volatile unsigned long * p);



/*

 * Little endian assembly bitops.  nr = 0 -> byte 0 bit 0.

 */

extern void _set_bit_le(int nr, volatile unsigned long * p);

extern void _clear_bit_le(int nr, volatile unsigned long * p);

extern void _change_bit_le(int nr, volatile unsigned long * p);

extern int _test_and_set_bit_le(int nr, volatile unsigned long * p);

extern int _test_and_clear_bit_le(int nr, volatile unsigned long * p);

extern int _test_and_change_bit_le(int nr, volatile unsigned long * p);

extern int _find_first_zero_bit_le(const void * p, unsigned size);

extern int _find_next_zero_bit_le(const void * p, int size, int offset);

extern int _find_first_bit_le(const unsigned long *p, unsigned size);
@@ -165,12 +169,6 @@ extern int _find_next_bit_le(const unsigned long *p, int size, int offset); 
/*

 * Big endian assembly bitops.  nr = 0 -> byte 3 bit 0.

 */

extern void _set_bit_be(int nr, volatile unsigned long * p);

extern void _clear_bit_be(int nr, volatile unsigned long * p);

extern void _change_bit_be(int nr, volatile unsigned long * p);

extern int _test_and_set_bit_be(int nr, volatile unsigned long * p);

extern int _test_and_clear_bit_be(int nr, volatile unsigned long * p);

extern int _test_and_change_bit_be(int nr, volatile unsigned long * p);

extern int _find_first_zero_bit_be(const void * p, unsigned size);

extern int _find_next_zero_bit_be(const void * p, int size, int offset);

extern int _find_first_bit_be(const unsigned long *p, unsigned size);
@@ -180,33 +178,26 @@ extern int _find_next_bit_be(const unsigned long *p, int size, int offset); 
/*

 * The __* form of bitops are non-atomic and may be reordered.

 */

#define	ATOMIC_BITOP_LE(name,nr,p)		\

	(__builtin_constant_p(nr) ?		\

	 ____atomic_##name(nr, p) :		\

	 _##name##_le(nr,p))



#define	ATOMIC_BITOP_BE(name,nr,p)		\

	(__builtin_constant_p(nr) ?		\

	 ____atomic_##name(nr, p) :		\

	 _##name##_be(nr,p))

#define ATOMIC_BITOP(name,nr,p)			\

	(__builtin_constant_p(nr) ? ____atomic_##name(nr, p) : _##name(nr,p))

#else

#define ATOMIC_BITOP_LE(name,nr,p)	_##name##_le(nr,p)

#define ATOMIC_BITOP_BE(name,nr,p)	_##name##_be(nr,p)

#define ATOMIC_BITOP(name,nr,p)		_##name(nr,p)

#endif



#define NONATOMIC_BITOP(name,nr,p)		\

	(____nonatomic_##name(nr, p))

/*

 * Native endian atomic definitions.

 */

#define set_bit(nr,p)			ATOMIC_BITOP(set_bit,nr,p)

#define clear_bit(nr,p)			ATOMIC_BITOP(clear_bit,nr,p)

#define change_bit(nr,p)		ATOMIC_BITOP(change_bit,nr,p)

#define test_and_set_bit(nr,p)		ATOMIC_BITOP(test_and_set_bit,nr,p)

#define test_and_clear_bit(nr,p)	ATOMIC_BITOP(test_and_clear_bit,nr,p)

#define test_and_change_bit(nr,p)	ATOMIC_BITOP(test_and_change_bit,nr,p)



#ifndef __ARMEB__

/*

 * These are the little endian, atomic definitions.

 */

#define set_bit(nr,p)			ATOMIC_BITOP_LE(set_bit,nr,p)

#define clear_bit(nr,p)			ATOMIC_BITOP_LE(clear_bit,nr,p)

#define change_bit(nr,p)		ATOMIC_BITOP_LE(change_bit,nr,p)

#define test_and_set_bit(nr,p)		ATOMIC_BITOP_LE(test_and_set_bit,nr,p)

#define test_and_clear_bit(nr,p)	ATOMIC_BITOP_LE(test_and_clear_bit,nr,p)

#define test_and_change_bit(nr,p)	ATOMIC_BITOP_LE(test_and_change_bit,nr,p)

#define find_first_zero_bit(p,sz)	_find_first_zero_bit_le(p,sz)

#define find_next_zero_bit(p,sz,off)	_find_next_zero_bit_le(p,sz,off)

#define find_first_bit(p,sz)		_find_first_bit_le(p,sz)
@@ -215,16 +206,9 @@ extern int _find_next_bit_be(const unsigned long *p, int size, int offset); 
#define WORD_BITOFF_TO_LE(x)		((x))



#else



/*

 * These are the big endian, atomic definitions.

 */

#define set_bit(nr,p)			ATOMIC_BITOP_BE(set_bit,nr,p)

#define clear_bit(nr,p)			ATOMIC_BITOP_BE(clear_bit,nr,p)

#define change_bit(nr,p)		ATOMIC_BITOP_BE(change_bit,nr,p)

#define test_and_set_bit(nr,p)		ATOMIC_BITOP_BE(test_and_set_bit,nr,p)

#define test_and_clear_bit(nr,p)	ATOMIC_BITOP_BE(test_and_clear_bit,nr,p)

#define test_and_change_bit(nr,p)	ATOMIC_BITOP_BE(test_and_change_bit,nr,p)

#define find_first_zero_bit(p,sz)	_find_first_zero_bit_be(p,sz)

#define find_next_zero_bit(p,sz,off)	_find_next_zero_bit_be(p,sz,off)

#define find_first_bit(p,sz)		_find_first_bit_be(p,sz)

arch/arm/kernel/armksyms.c

+6 −12
Original line number Diff line number Diff line
@@ -140,24 +140,18 @@ EXPORT_SYMBOL(__aeabi_ulcmp); 
#endif



	/* bitops */

EXPORT_SYMBOL(_set_bit_le);

EXPORT_SYMBOL(_test_and_set_bit_le);

EXPORT_SYMBOL(_clear_bit_le);

EXPORT_SYMBOL(_test_and_clear_bit_le);

EXPORT_SYMBOL(_change_bit_le);

EXPORT_SYMBOL(_test_and_change_bit_le);

EXPORT_SYMBOL(_set_bit);

EXPORT_SYMBOL(_test_and_set_bit);

EXPORT_SYMBOL(_clear_bit);

EXPORT_SYMBOL(_test_and_clear_bit);

EXPORT_SYMBOL(_change_bit);

EXPORT_SYMBOL(_test_and_change_bit);

EXPORT_SYMBOL(_find_first_zero_bit_le);

EXPORT_SYMBOL(_find_next_zero_bit_le);

EXPORT_SYMBOL(_find_first_bit_le);

EXPORT_SYMBOL(_find_next_bit_le);



#ifdef __ARMEB__

EXPORT_SYMBOL(_set_bit_be);

EXPORT_SYMBOL(_test_and_set_bit_be);

EXPORT_SYMBOL(_clear_bit_be);

EXPORT_SYMBOL(_test_and_clear_bit_be);

EXPORT_SYMBOL(_change_bit_be);

EXPORT_SYMBOL(_test_and_change_bit_be);

EXPORT_SYMBOL(_find_first_zero_bit_be);

EXPORT_SYMBOL(_find_next_zero_bit_be);

EXPORT_SYMBOL(_find_first_bit_be);

arch/arm/lib/bitops.h

+20 −18
Original line number Diff line number Diff line 


#if __LINUX_ARM_ARCH__ >= 6 && defined(CONFIG_CPU_32v6K)

#if __LINUX_ARM_ARCH__ >= 6

	.macro	bitop, instr

	ands	ip, r1, #3

	strneb	r1, [ip]		@ assert word-aligned

	mov	r2, #1

	and	r3, r0, #7		@ Get bit offset

	add	r1, r1, r0, lsr #3	@ Get byte offset

	and	r3, r0, #31		@ Get bit offset

	mov	r0, r0, lsr #5

	add	r1, r1, r0, lsl #2	@ Get word offset

	mov	r3, r2, lsl r3

1:	ldrexb	r2, [r1]

1:	ldrex	r2, [r1]

	\instr	r2, r2, r3

	strexb	r0, r2, [r1]

	strex	r0, r2, [r1]

	cmp	r0, #0

	bne	1b

	mov	pc, lr
@@ -18,15 +18,16 @@ 
	.macro	testop, instr, store

	ands	ip, r1, #3

	strneb	r1, [ip]		@ assert word-aligned

	and	r3, r0, #7		@ Get bit offset

	mov	r2, #1

	add	r1, r1, r0, lsr #3	@ Get byte offset

	and	r3, r0, #31		@ Get bit offset

	mov	r0, r0, lsr #5

	add	r1, r1, r0, lsl #2	@ Get word offset

	mov	r3, r2, lsl r3		@ create mask

	smp_dmb

1:	ldrexb	r2, [r1]

1:	ldrex	r2, [r1]

	ands	r0, r2, r3		@ save old value of bit

	\instr	r2, r2, r3		@ toggle bit

	strexb	ip, r2, [r1]

	strex	ip, r2, [r1]

	cmp	ip, #0

	bne	1b

	smp_dmb
@@ -38,13 +39,14 @@ 
	.macro	bitop, instr

	ands	ip, r1, #3

	strneb	r1, [ip]		@ assert word-aligned

	and	r2, r0, #7

	and	r2, r0, #31

	mov	r0, r0, lsr #5

	mov	r3, #1

	mov	r3, r3, lsl r2

	save_and_disable_irqs ip

	ldrb	r2, [r1, r0, lsr #3]

	ldr	r2, [r1, r0, lsl #2]

	\instr	r2, r2, r3

	strb	r2, [r1, r0, lsr #3]

	str	r2, [r1, r0, lsl #2]

	restore_irqs ip

	mov	pc, lr

	.endm
@@ -60,11 +62,11 @@ 
	.macro	testop, instr, store

	ands	ip, r1, #3

	strneb	r1, [ip]		@ assert word-aligned

	add	r1, r1, r0, lsr #3

	and	r3, r0, #7

	mov	r0, #1

	and	r3, r0, #31

	mov	r0, r0, lsr #5

	save_and_disable_irqs ip

	ldrb	r2, [r1]

	ldr	r2, [r1, r0, lsl #2]!

	mov	r0, #1

	tst	r2, r0, lsl r3

	\instr	r2, r2, r0, lsl r3

	\store	r2, [r1]

arch/arm/lib/changebit.S

+2 −8
Original line number Diff line number Diff line
@@ -12,12 +12,6 @@ 
#include "bitops.h"

                .text



/* Purpose  : Function to change a bit

 * Prototype: int change_bit(int bit, void *addr)

 */

ENTRY(_change_bit_be)

		eor	r0, r0, #0x18		@ big endian byte ordering

ENTRY(_change_bit_le)

ENTRY(_change_bit)

	bitop	eor

ENDPROC(_change_bit_be)

ENDPROC(_change_bit_le)
 
ENDPROC(_change_bit)

arch/arm/lib/clearbit.S

+2 −9
Original line number Diff line number Diff line
@@ -12,13 +12,6 @@ 
#include "bitops.h"

                .text



/*

 * Purpose  : Function to clear a bit

 * Prototype: int clear_bit(int bit, void *addr)

 */

ENTRY(_clear_bit_be)

		eor	r0, r0, #0x18		@ big endian byte ordering

ENTRY(_clear_bit_le)

ENTRY(_clear_bit)

	bitop	bic

ENDPROC(_clear_bit_be)

ENDPROC(_clear_bit_le)
 
ENDPROC(_clear_bit)

CRA Git | Maintained and supported by SUSTech CRA and CCSE