Commit 9b3bb86a authored by David S. Miller's avatar David S. Miller
Browse files

sparc64: Make rwsems 64-bit. 



Basically tip-off the powerpc code, use a 64-bit type and atomic64_t
interfaces for the implementation.

This gets us off of the by-hand asm code I wrote, which frankly I
think probably ruins I-cache hit rates.

The idea was the keep the call chains less deep, but anything taking
the rw-semaphores probably is also calling other stuff and therefore
already has allocated a stack-frame.  So no real stack frame savings
ever.

Ben H. has posted patches to make powerpc use 64-bit too and with some
abstractions we can probably use a shared header file somewhere.

With suggestions from Sam Ravnborg.

Signed-off-by: default avatarDavid S. Miller <davem@davemloft.net>
parent b10f997b

arch/sparc/include/asm/rwsem-const.h

deleted100644 → 0
+0 −12
Original line number Diff line number Diff line 
/* rwsem-const.h: RW semaphore counter constants.  */

#ifndef _SPARC64_RWSEM_CONST_H

#define _SPARC64_RWSEM_CONST_H



#define RWSEM_UNLOCKED_VALUE		0x00000000

#define RWSEM_ACTIVE_BIAS		0x00000001

#define RWSEM_ACTIVE_MASK		0x0000ffff

#define RWSEM_WAITING_BIAS		(-0x00010000)

#define RWSEM_ACTIVE_READ_BIAS		RWSEM_ACTIVE_BIAS

#define RWSEM_ACTIVE_WRITE_BIAS		(RWSEM_WAITING_BIAS + RWSEM_ACTIVE_BIAS)



#endif /* _SPARC64_RWSEM_CONST_H */

arch/sparc/include/asm/rwsem.h

+103 −17
Original line number Diff line number Diff line
@@ -15,12 +15,17 @@ 


#include <linux/list.h>

#include <linux/spinlock.h>

#include <asm/rwsem-const.h>



struct rwsem_waiter;



struct rw_semaphore {

	signed int count;

	signed long			count;

#define RWSEM_UNLOCKED_VALUE		0x00000000L

#define RWSEM_ACTIVE_BIAS		0x00000001L

#define RWSEM_ACTIVE_MASK		0xffffffffL

#define RWSEM_WAITING_BIAS		(-RWSEM_ACTIVE_MASK-1)

#define RWSEM_ACTIVE_READ_BIAS		RWSEM_ACTIVE_BIAS

#define RWSEM_ACTIVE_WRITE_BIAS		(RWSEM_WAITING_BIAS + RWSEM_ACTIVE_BIAS)

	spinlock_t			wait_lock;

	struct list_head		wait_list;

#ifdef CONFIG_DEBUG_LOCK_ALLOC
@@ -41,6 +46,11 @@ struct rw_semaphore { 
#define DECLARE_RWSEM(name) \

	struct rw_semaphore name = __RWSEM_INITIALIZER(name)



extern struct rw_semaphore *rwsem_down_read_failed(struct rw_semaphore *sem);

extern struct rw_semaphore *rwsem_down_write_failed(struct rw_semaphore *sem);

extern struct rw_semaphore *rwsem_wake(struct rw_semaphore *sem);

extern struct rw_semaphore *rwsem_downgrade_wake(struct rw_semaphore *sem);



extern void __init_rwsem(struct rw_semaphore *sem, const char *name,

			 struct lock_class_key *key);
@@ -51,27 +61,103 @@ do { \ 
	__init_rwsem((sem), #sem, &__key);			\

} while (0)



extern void __down_read(struct rw_semaphore *sem);

extern int __down_read_trylock(struct rw_semaphore *sem);

extern void __down_write(struct rw_semaphore *sem);

extern int __down_write_trylock(struct rw_semaphore *sem);

extern void __up_read(struct rw_semaphore *sem);

extern void __up_write(struct rw_semaphore *sem);

extern void __downgrade_write(struct rw_semaphore *sem);

/*

 * lock for reading

 */

static inline void __down_read(struct rw_semaphore *sem)

{

	if (unlikely(atomic64_inc_return((atomic64_t *)(&sem->count)) <= 0L))

		rwsem_down_read_failed(sem);

}



static inline int __down_read_trylock(struct rw_semaphore *sem)

{

	long tmp;



	while ((tmp = sem->count) >= 0L) {

		if (tmp == cmpxchg(&sem->count, tmp,

				   tmp + RWSEM_ACTIVE_READ_BIAS)) {

			return 1;

		}

	}

	return 0;

}



/*

 * lock for writing

 */

static inline void __down_write_nested(struct rw_semaphore *sem, int subclass)

{

	__down_write(sem);

	long tmp;



	tmp = atomic64_add_return(RWSEM_ACTIVE_WRITE_BIAS,

				  (atomic64_t *)(&sem->count));

	if (unlikely(tmp != RWSEM_ACTIVE_WRITE_BIAS))

		rwsem_down_write_failed(sem);

}



static inline void __down_write(struct rw_semaphore *sem)

{

	__down_write_nested(sem, 0);

}



static inline int __down_write_trylock(struct rw_semaphore *sem)

{

	long tmp;



	tmp = cmpxchg(&sem->count, RWSEM_UNLOCKED_VALUE,

		      RWSEM_ACTIVE_WRITE_BIAS);

	return tmp == RWSEM_UNLOCKED_VALUE;

}



/*

 * unlock after reading

 */

static inline void __up_read(struct rw_semaphore *sem)

{

	long tmp;



	tmp = atomic64_dec_return((atomic64_t *)(&sem->count));

	if (unlikely(tmp < -1L && (tmp & RWSEM_ACTIVE_MASK) == 0L))

		rwsem_wake(sem);

}



static inline int rwsem_atomic_update(int delta, struct rw_semaphore *sem)

/*

 * unlock after writing

 */

static inline void __up_write(struct rw_semaphore *sem)

{

	return atomic_add_return(delta, (atomic_t *)(&sem->count));

	if (unlikely(atomic64_sub_return(RWSEM_ACTIVE_WRITE_BIAS,

					 (atomic64_t *)(&sem->count)) < 0L))

		rwsem_wake(sem);

}



static inline void rwsem_atomic_add(int delta, struct rw_semaphore *sem)

/*

 * implement atomic add functionality

 */

static inline void rwsem_atomic_add(long delta, struct rw_semaphore *sem)

{

	atomic64_add(delta, (atomic64_t *)(&sem->count));

}



/*

 * downgrade write lock to read lock

 */

static inline void __downgrade_write(struct rw_semaphore *sem)

{

	long tmp;



	tmp = atomic64_add_return(-RWSEM_WAITING_BIAS, (atomic64_t *)(&sem->count));

	if (tmp < 0L)

		rwsem_downgrade_wake(sem);

}



/*

 * implement exchange and add functionality

 */

static inline long rwsem_atomic_update(long delta, struct rw_semaphore *sem)

{

	atomic_add(delta, (atomic_t *)(&sem->count));

	return atomic64_add_return(delta, (atomic64_t *)(&sem->count));

}



static inline int rwsem_is_locked(struct rw_semaphore *sem)

arch/sparc/lib/Makefile

+1 −1
Original line number Diff line number Diff line
@@ -15,7 +15,7 @@ lib-$(CONFIG_SPARC32) += divdi3.o udivdi3.o 
lib-$(CONFIG_SPARC32) += copy_user.o locks.o

lib-y                 += atomic_$(BITS).o

lib-$(CONFIG_SPARC32) += lshrdi3.o ashldi3.o

lib-y                 += rwsem_$(BITS).o

lib-$(CONFIG_SPARC32) += rwsem_32.o

lib-$(CONFIG_SPARC32) += muldi3.o bitext.o cmpdi2.o



lib-$(CONFIG_SPARC64) += copy_page.o clear_page.o bzero.o

arch/sparc/lib/rwsem_64.S

deleted100644 → 0
+0 −163
Original line number Diff line number Diff line 
/* rwsem.S: RW semaphore assembler.

 *

 * Written by David S. Miller (davem@redhat.com), 2001.

 * Derived from asm-i386/rwsem.h

 */



#include <asm/rwsem-const.h>



	.section	.sched.text, "ax"



	.globl		__down_read

__down_read:

1:	lduw		[%o0], %g1

	add		%g1, 1, %g7

	cas		[%o0], %g1, %g7

	cmp		%g1, %g7

	bne,pn		%icc, 1b

	 add		%g7, 1, %g7

	cmp		%g7, 0

	bl,pn		%icc, 3f

	 nop

2:

	retl

	 nop

3:

	save		%sp, -192, %sp

	call		rwsem_down_read_failed

	 mov		%i0, %o0

	ret

	 restore

	.size		__down_read, .-__down_read



	.globl		__down_read_trylock

__down_read_trylock:

1:	lduw		[%o0], %g1

	add		%g1, 1, %g7

	cmp		%g7, 0

	bl,pn		%icc, 2f

	 mov		0, %o1

	cas		[%o0], %g1, %g7

	cmp		%g1, %g7

	bne,pn		%icc, 1b

	 mov		1, %o1

2:	retl

	 mov		%o1, %o0

	.size		__down_read_trylock, .-__down_read_trylock



	.globl		__down_write

__down_write:

	sethi		%hi(RWSEM_ACTIVE_WRITE_BIAS), %g1

	or		%g1, %lo(RWSEM_ACTIVE_WRITE_BIAS), %g1

1:

	lduw		[%o0], %g3

	add		%g3, %g1, %g7

	cas		[%o0], %g3, %g7

	cmp		%g3, %g7

	bne,pn		%icc, 1b

	 cmp		%g7, 0

	bne,pn		%icc, 3f

	 nop

2:	retl

	 nop

3:

	save		%sp, -192, %sp

	call		rwsem_down_write_failed

	 mov		%i0, %o0

	ret

	 restore

	.size		__down_write, .-__down_write



	.globl		__down_write_trylock

__down_write_trylock:

	sethi		%hi(RWSEM_ACTIVE_WRITE_BIAS), %g1

	or		%g1, %lo(RWSEM_ACTIVE_WRITE_BIAS), %g1

1:

	lduw		[%o0], %g3

	cmp		%g3, 0

	bne,pn		%icc, 2f

	 mov		0, %o1

	add		%g3, %g1, %g7

	cas		[%o0], %g3, %g7

	cmp		%g3, %g7

	bne,pn		%icc, 1b

	 mov		1, %o1

2:	retl

	 mov		%o1, %o0

	.size		__down_write_trylock, .-__down_write_trylock



	.globl		__up_read

__up_read:

1:

	lduw		[%o0], %g1

	sub		%g1, 1, %g7

	cas		[%o0], %g1, %g7

	cmp		%g1, %g7

	bne,pn		%icc, 1b

	 cmp		%g7, 0

	bl,pn		%icc, 3f

	 nop

2:	retl

	 nop

3:	sethi		%hi(RWSEM_ACTIVE_MASK), %g1

	sub		%g7, 1, %g7

	or		%g1, %lo(RWSEM_ACTIVE_MASK), %g1

	andcc		%g7, %g1, %g0

	bne,pn		%icc, 2b

	 nop

	save		%sp, -192, %sp

	call		rwsem_wake

	 mov		%i0, %o0

	ret

	 restore

	.size		__up_read, .-__up_read



	.globl		__up_write

__up_write:

	sethi		%hi(RWSEM_ACTIVE_WRITE_BIAS), %g1

	or		%g1, %lo(RWSEM_ACTIVE_WRITE_BIAS), %g1

1:

	lduw		[%o0], %g3

	sub		%g3, %g1, %g7

	cas		[%o0], %g3, %g7

	cmp		%g3, %g7

	bne,pn		%icc, 1b

	 sub		%g7, %g1, %g7

	cmp		%g7, 0

	bl,pn		%icc, 3f

	 nop

2:

	retl

	 nop

3:

	save		%sp, -192, %sp

	call		rwsem_wake

	 mov		%i0, %o0

	ret

	 restore

	.size		__up_write, .-__up_write



	.globl		__downgrade_write

__downgrade_write:

	sethi		%hi(RWSEM_WAITING_BIAS), %g1

	or		%g1, %lo(RWSEM_WAITING_BIAS), %g1

1:

	lduw		[%o0], %g3

	sub		%g3, %g1, %g7

	cas		[%o0], %g3, %g7

	cmp		%g3, %g7

	bne,pn		%icc, 1b

	 sub		%g7, %g1, %g7

	cmp		%g7, 0

	bl,pn		%icc, 3f

	 nop

2:

	retl

	 nop

3:

	save		%sp, -192, %sp

	call		rwsem_downgrade_wake

	 mov		%i0, %o0

	ret

	 restore

	.size		__downgrade_write, .-__downgrade_write

CRA Git | Maintained and supported by SUSTech CRA and CCSE