Commit 7a215f89 authored by Davidlohr Bueso's avatar Davidlohr Bueso Committed by Ingo Molnar
Browse files

locking/rwsem: Set lock ownership ASAP 



In order to optimize the spinning step, we need to set the lock
owner as soon as the lock is acquired; after a successful counter
cmpxchg operation, that is. This is particularly useful as rwsems
need to set the owner to nil for readers, so there is a greater
chance of falling out of the spinning. Currently we only set the
owner much later in the game, in the more generic level -- latency
can be specially bad when waiting for a node->next pointer when
releasing the osq in up_write calls.

As such, update the owner inside rwsem_try_write_lock (when the
lock is obtained after blocking) and rwsem_try_write_lock_unqueued
(when the lock is obtained while spinning). This requires creating
a new internal rwsem.h header to share the owner related calls.

Also cleanup some headers for mutex and rwsem.

Suggested-by: default avatarPeter Zijlstra <peterz@infradead.org>
Signed-off-by: default avatarDavidlohr Bueso <dbueso@suse.de>
Signed-off-by: default avatarPeter Zijlstra (Intel) <peterz@infradead.org>
Cc: Jason Low <jason.low2@hp.com>
Cc: Linus Torvalds <torvalds@linux-foundation.org>
Cc: Michel Lespinasse <walken@google.com>
Cc: Paul E. McKenney <paulmck@linux.vnet.ibm.com>
Cc: Tim Chen <tim.c.chen@linux.intel.com>
Link: http://lkml.kernel.org/r/1422609267-15102-4-git-send-email-dave@stgolabs.net


Signed-off-by: default avatarIngo Molnar <mingo@kernel.org>
parent 49e4b2bc

kernel/locking/mutex.c

+1 −1
Original line number Diff line number Diff line
@@ -25,7 +25,7 @@ 
#include <linux/spinlock.h>

#include <linux/interrupt.h>

#include <linux/debug_locks.h>

#include "mcs_spinlock.h"

#include <linux/osq_lock.h>



/*

 * In the DEBUG case we are using the "NULL fastpath" for mutexes,

kernel/locking/rwsem-xadd.c

+6 −2
Original line number Diff line number Diff line
@@ -14,8 +14,9 @@ 
#include <linux/init.h>

#include <linux/export.h>

#include <linux/sched/rt.h>

#include <linux/osq_lock.h>



#include "mcs_spinlock.h"

#include "rwsem.h"



/*

 * Guide to the rw_semaphore's count field for common values.
@@ -265,6 +266,7 @@ static inline bool rwsem_try_write_lock(long count, struct rw_semaphore *sem) 
		    RWSEM_ACTIVE_WRITE_BIAS) == RWSEM_WAITING_BIAS) {

		if (!list_is_singular(&sem->wait_list))

			rwsem_atomic_update(RWSEM_WAITING_BIAS, sem);

		rwsem_set_owner(sem);

		return true;

	}
@@ -284,8 +286,10 @@ static inline bool rwsem_try_write_lock_unqueued(struct rw_semaphore *sem) 
			return false;



		old = cmpxchg(&sem->count, count, count + RWSEM_ACTIVE_WRITE_BIAS);

		if (old == count)

		if (old == count) {

			rwsem_set_owner(sem);

			return true;

		}



		count = old;

	}

kernel/locking/rwsem.c

+1 −21
Original line number Diff line number Diff line
@@ -9,29 +9,9 @@ 
#include <linux/sched.h>

#include <linux/export.h>

#include <linux/rwsem.h>



#include <linux/atomic.h>



#ifdef CONFIG_RWSEM_SPIN_ON_OWNER

static inline void rwsem_set_owner(struct rw_semaphore *sem)

{

	sem->owner = current;

}



static inline void rwsem_clear_owner(struct rw_semaphore *sem)

{

	sem->owner = NULL;

}



#else

static inline void rwsem_set_owner(struct rw_semaphore *sem)

{

}



static inline void rwsem_clear_owner(struct rw_semaphore *sem)

{

}

#endif

#include "rwsem.h"



/*

 * lock for reading

kernel/locking/rwsem.h

0 → 100644
+20 −0
Original line number Diff line number Diff line 
#ifdef CONFIG_RWSEM_SPIN_ON_OWNER

static inline void rwsem_set_owner(struct rw_semaphore *sem)

{

	sem->owner = current;

}



static inline void rwsem_clear_owner(struct rw_semaphore *sem)

{

	sem->owner = NULL;

}



#else

static inline void rwsem_set_owner(struct rw_semaphore *sem)

{

}



static inline void rwsem_clear_owner(struct rw_semaphore *sem)

{

}

#endif

CRA Git | Maintained and supported by SUSTech CRA and CCSE