Commit 46ad0840 authored by Waiman Long's avatar Waiman Long Committed by Ingo Molnar
Browse files

locking/rwsem: Remove arch specific rwsem files 



As the generic rwsem-xadd code is using the appropriate acquire and
release versions of the atomic operations, the arch specific rwsem.h
files will not be that much faster than the generic code as long as the
atomic functions are properly implemented. So we can remove those arch
specific rwsem.h and stop building asm/rwsem.h to reduce maintenance
effort.

Currently, only x86, alpha and ia64 have implemented architecture
specific fast paths. I don't have access to alpha and ia64 systems for
testing, but they are legacy systems that are not likely to be updated
to the latest kernel anyway.

By using a rwsem microbenchmark, the total locking rates on a 4-socket
56-core 112-thread x86-64 system before and after the patch were as
follows (mixed means equal # of read and write locks):

                      Before Patch              After Patch
   # of Threads  wlock   rlock   mixed     wlock   rlock   mixed
   ------------  -----   -----   -----     -----   -----   -----
        1        29,201  30,143  29,458    28,615  30,172  29,201
        2         6,807  13,299   1,171     7,725  15,025   1,804
        4         6,504  12,755   1,520     7,127  14,286   1,345
        8         6,762  13,412     764     6,826  13,652     726
       16         6,693  15,408     662     6,599  15,938     626
       32         6,145  15,286     496     5,549  15,487     511
       64         5,812  15,495      60     5,858  15,572      60

There were some run-to-run variations for the multi-thread tests. For
x86-64, using the generic C code fast path seems to be a little bit
faster than the assembly version with low lock contention.  Looking at
the assembly version of the fast paths, there are assembly to/from C
code wrappers that save and restore all the callee-clobbered registers
(7 registers on x86-64). The assembly generated from the generic C
code doesn't need to do that. That may explain the slight performance
gain here.

The generic asm rwsem.h can also be merged into kernel/locking/rwsem.h
with no code change as no other code other than those under
kernel/locking needs to access the internal rwsem macros and functions.

Signed-off-by: default avatarWaiman Long <longman@redhat.com>
Signed-off-by: default avatarPeter Zijlstra (Intel) <peterz@infradead.org>
Acked-by: default avatarLinus Torvalds <torvalds@linux-foundation.org>
Cc: Andrew Morton <akpm@linux-foundation.org>
Cc: Arnd Bergmann <arnd@arndb.de>
Cc: Borislav Petkov <bp@alien8.de>
Cc: Davidlohr Bueso <dave@stgolabs.net>
Cc: H. Peter Anvin <hpa@zytor.com>
Cc: Paul E. McKenney <paulmck@linux.vnet.ibm.com>
Cc: Peter Zijlstra <peterz@infradead.org>
Cc: Thomas Gleixner <tglx@linutronix.de>
Cc: Tim Chen <tim.c.chen@linux.intel.com>
Cc: Will Deacon <will.deacon@arm.com>
Cc: linux-arm-kernel@lists.infradead.org
Cc: linux-c6x-dev@linux-c6x.org
Cc: linux-m68k@lists.linux-m68k.org
Cc: linux-riscv@lists.infradead.org
Cc: linux-um@lists.infradead.org
Cc: linux-xtensa@linux-xtensa.org
Cc: linuxppc-dev@lists.ozlabs.org
Cc: nios2-dev@lists.rocketboards.org
Cc: openrisc@lists.librecores.org
Cc: uclinux-h8-devel@lists.sourceforge.jp
Link: https://lkml.kernel.org/r/20190322143008.21313-2-longman@redhat.com


Signed-off-by: default avatarIngo Molnar <mingo@kernel.org>
parent a1247d06

MAINTAINERS

+0 −1
Original line number Diff line number Diff line
@@ -9098,7 +9098,6 @@ F: arch/*/include/asm/spinlock*.h 
F:	include/linux/rwlock*.h

F:	include/linux/mutex*.h

F:	include/linux/rwsem*.h

F:	arch/*/include/asm/rwsem.h

F:	include/linux/seqlock.h

F:	lib/locking*.[ch]

F:	kernel/locking/

arch/alpha/include/asm/rwsem.h

deleted100644 → 0
+0 −211
Original line number Diff line number Diff line 
/* SPDX-License-Identifier: GPL-2.0 */

#ifndef _ALPHA_RWSEM_H

#define _ALPHA_RWSEM_H



/*

 * Written by Ivan Kokshaysky <ink@jurassic.park.msu.ru>, 2001.

 * Based on asm-alpha/semaphore.h and asm-i386/rwsem.h

 */



#ifndef _LINUX_RWSEM_H

#error "please don't include asm/rwsem.h directly, use linux/rwsem.h instead"

#endif



#ifdef __KERNEL__



#include <linux/compiler.h>



#define RWSEM_UNLOCKED_VALUE		0x0000000000000000L

#define RWSEM_ACTIVE_BIAS		0x0000000000000001L

#define RWSEM_ACTIVE_MASK		0x00000000ffffffffL

#define RWSEM_WAITING_BIAS		(-0x0000000100000000L)

#define RWSEM_ACTIVE_READ_BIAS		RWSEM_ACTIVE_BIAS

#define RWSEM_ACTIVE_WRITE_BIAS		(RWSEM_WAITING_BIAS + RWSEM_ACTIVE_BIAS)



static inline int ___down_read(struct rw_semaphore *sem)

{

	long oldcount;

#ifndef	CONFIG_SMP

	oldcount = sem->count.counter;

	sem->count.counter += RWSEM_ACTIVE_READ_BIAS;

#else

	long temp;

	__asm__ __volatile__(

	"1:	ldq_l	%0,%1\n"

	"	addq	%0,%3,%2\n"

	"	stq_c	%2,%1\n"

	"	beq	%2,2f\n"

	"	mb\n"

	".subsection 2\n"

	"2:	br	1b\n"

	".previous"

	:"=&r" (oldcount), "=m" (sem->count), "=&r" (temp)

	:"Ir" (RWSEM_ACTIVE_READ_BIAS), "m" (sem->count) : "memory");

#endif

	return (oldcount < 0);

}



static inline void __down_read(struct rw_semaphore *sem)

{

	if (unlikely(___down_read(sem)))

		rwsem_down_read_failed(sem);

}



static inline int __down_read_killable(struct rw_semaphore *sem)

{

	if (unlikely(___down_read(sem)))

		if (IS_ERR(rwsem_down_read_failed_killable(sem)))

			return -EINTR;



	return 0;

}



/*

 * trylock for reading -- returns 1 if successful, 0 if contention

 */

static inline int __down_read_trylock(struct rw_semaphore *sem)

{

	long old, new, res;



	res = atomic_long_read(&sem->count);

	do {

		new = res + RWSEM_ACTIVE_READ_BIAS;

		if (new <= 0)

			break;

		old = res;

		res = atomic_long_cmpxchg(&sem->count, old, new);

	} while (res != old);

	return res >= 0 ? 1 : 0;

}



static inline long ___down_write(struct rw_semaphore *sem)

{

	long oldcount;

#ifndef	CONFIG_SMP

	oldcount = sem->count.counter;

	sem->count.counter += RWSEM_ACTIVE_WRITE_BIAS;

#else

	long temp;

	__asm__ __volatile__(

	"1:	ldq_l	%0,%1\n"

	"	addq	%0,%3,%2\n"

	"	stq_c	%2,%1\n"

	"	beq	%2,2f\n"

	"	mb\n"

	".subsection 2\n"

	"2:	br	1b\n"

	".previous"

	:"=&r" (oldcount), "=m" (sem->count), "=&r" (temp)

	:"Ir" (RWSEM_ACTIVE_WRITE_BIAS), "m" (sem->count) : "memory");

#endif

	return oldcount;

}



static inline void __down_write(struct rw_semaphore *sem)

{

	if (unlikely(___down_write(sem)))

		rwsem_down_write_failed(sem);

}



static inline int __down_write_killable(struct rw_semaphore *sem)

{

	if (unlikely(___down_write(sem))) {

		if (IS_ERR(rwsem_down_write_failed_killable(sem)))

			return -EINTR;

	}



	return 0;

}



/*

 * trylock for writing -- returns 1 if successful, 0 if contention

 */

static inline int __down_write_trylock(struct rw_semaphore *sem)

{

	long ret = atomic_long_cmpxchg(&sem->count, RWSEM_UNLOCKED_VALUE,

			   RWSEM_ACTIVE_WRITE_BIAS);

	if (ret == RWSEM_UNLOCKED_VALUE)

		return 1;

	return 0;

}



static inline void __up_read(struct rw_semaphore *sem)

{

	long oldcount;

#ifndef	CONFIG_SMP

	oldcount = sem->count.counter;

	sem->count.counter -= RWSEM_ACTIVE_READ_BIAS;

#else

	long temp;

	__asm__ __volatile__(

	"	mb\n"

	"1:	ldq_l	%0,%1\n"

	"	subq	%0,%3,%2\n"

	"	stq_c	%2,%1\n"

	"	beq	%2,2f\n"

	".subsection 2\n"

	"2:	br	1b\n"

	".previous"

	:"=&r" (oldcount), "=m" (sem->count), "=&r" (temp)

	:"Ir" (RWSEM_ACTIVE_READ_BIAS), "m" (sem->count) : "memory");

#endif

	if (unlikely(oldcount < 0))

		if ((int)oldcount - RWSEM_ACTIVE_READ_BIAS == 0)

			rwsem_wake(sem);

}



static inline void __up_write(struct rw_semaphore *sem)

{

	long count;

#ifndef	CONFIG_SMP

	sem->count.counter -= RWSEM_ACTIVE_WRITE_BIAS;

	count = sem->count.counter;

#else

	long temp;

	__asm__ __volatile__(

	"	mb\n"

	"1:	ldq_l	%0,%1\n"

	"	subq	%0,%3,%2\n"

	"	stq_c	%2,%1\n"

	"	beq	%2,2f\n"

	"	subq	%0,%3,%0\n"

	".subsection 2\n"

	"2:	br	1b\n"

	".previous"

	:"=&r" (count), "=m" (sem->count), "=&r" (temp)

	:"Ir" (RWSEM_ACTIVE_WRITE_BIAS), "m" (sem->count) : "memory");

#endif

	if (unlikely(count))

		if ((int)count == 0)

			rwsem_wake(sem);

}



/*

 * downgrade write lock to read lock

 */

static inline void __downgrade_write(struct rw_semaphore *sem)

{

	long oldcount;

#ifndef	CONFIG_SMP

	oldcount = sem->count.counter;

	sem->count.counter -= RWSEM_WAITING_BIAS;

#else

	long temp;

	__asm__ __volatile__(

	"1:	ldq_l	%0,%1\n"

	"	addq	%0,%3,%2\n"

	"	stq_c	%2,%1\n"

	"	beq	%2,2f\n"

	"	mb\n"

	".subsection 2\n"

	"2:	br	1b\n"

	".previous"

	:"=&r" (oldcount), "=m" (sem->count), "=&r" (temp)

	:"Ir" (-RWSEM_WAITING_BIAS), "m" (sem->count) : "memory");

#endif

	if (unlikely(oldcount < 0))

		rwsem_downgrade_wake(sem);

}



#endif /* __KERNEL__ */

#endif /* _ALPHA_RWSEM_H */

arch/arm/include/asm/Kbuild

+0 −1
Original line number Diff line number Diff line
@@ -12,7 +12,6 @@ generic-y += mm-arch-hooks.h 
generic-y += msi.h

generic-y += parport.h

generic-y += preempt.h

generic-y += rwsem.h

generic-y += seccomp.h

generic-y += segment.h

generic-y += serial.h

arch/arm64/include/asm/Kbuild

+0 −1
Original line number Diff line number Diff line
@@ -16,7 +16,6 @@ generic-y += mm-arch-hooks.h 
generic-y += msi.h

generic-y += qrwlock.h

generic-y += qspinlock.h

generic-y += rwsem.h

generic-y += segment.h

generic-y += serial.h

generic-y += set_memory.h

arch/hexagon/include/asm/Kbuild

+0 −1
Original line number Diff line number Diff line
@@ -27,7 +27,6 @@ generic-y += mm-arch-hooks.h 
generic-y += pci.h

generic-y += percpu.h

generic-y += preempt.h

generic-y += rwsem.h

generic-y += sections.h

generic-y += segment.h

generic-y += serial.h

CRA Git | Maintained and supported by SUSTech CRA and CCSE