Commit 1751060e authored by Peter Zijlstra's avatar Peter Zijlstra Committed by Ingo Molnar
Browse files

locking/percpu-rwsem, lockdep: Make percpu-rwsem use its own lockdep_map 



As preparation for replacing the embedded rwsem, give percpu-rwsem its
own lockdep_map.

Signed-off-by: default avatarPeter Zijlstra (Intel) <peterz@infradead.org>
Signed-off-by: default avatarIngo Molnar <mingo@kernel.org>
Reviewed-by: default avatarDavidlohr Bueso <dbueso@suse.de>
Acked-by: default avatarWill Deacon <will@kernel.org>
Acked-by: default avatarWaiman Long <longman@redhat.com>
Tested-by: default avatarJuri Lelli <juri.lelli@redhat.com>
Link: https://lkml.kernel.org/r/20200131151539.927625541@infradead.org
parent 810507fe

include/linux/percpu-rwsem.h

+19 −10
Original line number Diff line number Diff line
@@ -15,8 +15,17 @@ struct percpu_rw_semaphore { 
	struct rw_semaphore	rw_sem; /* slowpath */

	struct rcuwait          writer; /* blocked writer */

	int			readers_block;

#ifdef CONFIG_DEBUG_LOCK_ALLOC

	struct lockdep_map	dep_map;

#endif

};



#ifdef CONFIG_DEBUG_LOCK_ALLOC

#define __PERCPU_RWSEM_DEP_MAP_INIT(lockname)	.dep_map = { .name = #lockname },

#else

#define __PERCPU_RWSEM_DEP_MAP_INIT(lockname)

#endif



#define __DEFINE_PERCPU_RWSEM(name, is_static)				\

static DEFINE_PER_CPU(unsigned int, __percpu_rwsem_rc_##name);		\

is_static struct percpu_rw_semaphore name = {				\
@@ -24,7 +33,9 @@ is_static struct percpu_rw_semaphore name = { \ 
	.read_count = &__percpu_rwsem_rc_##name,			\

	.rw_sem = __RWSEM_INITIALIZER(name.rw_sem),			\

	.writer = __RCUWAIT_INITIALIZER(name.writer),			\

	__PERCPU_RWSEM_DEP_MAP_INIT(name)				\

}



#define DEFINE_PERCPU_RWSEM(name)		\

	__DEFINE_PERCPU_RWSEM(name, /* not static */)

#define DEFINE_STATIC_PERCPU_RWSEM(name)	\
@@ -37,7 +48,7 @@ static inline void percpu_down_read(struct percpu_rw_semaphore *sem) 
{

	might_sleep();



	rwsem_acquire_read(&sem->rw_sem.dep_map, 0, 0, _RET_IP_);

	rwsem_acquire_read(&sem->dep_map, 0, 0, _RET_IP_);



	preempt_disable();

	/*
@@ -76,13 +87,15 @@ static inline int percpu_down_read_trylock(struct percpu_rw_semaphore *sem) 
	 */



	if (ret)

		rwsem_acquire_read(&sem->rw_sem.dep_map, 0, 1, _RET_IP_);

		rwsem_acquire_read(&sem->dep_map, 0, 1, _RET_IP_);



	return ret;

}



static inline void percpu_up_read(struct percpu_rw_semaphore *sem)

{

	rwsem_release(&sem->dep_map, _RET_IP_);



	preempt_disable();

	/*

	 * Same as in percpu_down_read().
@@ -92,8 +105,6 @@ static inline void percpu_up_read(struct percpu_rw_semaphore *sem) 
	else

		__percpu_up_read(sem); /* Unconditional memory barrier */

	preempt_enable();



	rwsem_release(&sem->rw_sem.dep_map, _RET_IP_);

}



extern void percpu_down_write(struct percpu_rw_semaphore *);
@@ -110,15 +121,13 @@ extern void percpu_free_rwsem(struct percpu_rw_semaphore *); 
	__percpu_init_rwsem(sem, #sem, &rwsem_key);		\

})



#define percpu_rwsem_is_held(sem) lockdep_is_held(&(sem)->rw_sem)



#define percpu_rwsem_assert_held(sem)				\

	lockdep_assert_held(&(sem)->rw_sem)

#define percpu_rwsem_is_held(sem)	lockdep_is_held(sem)

#define percpu_rwsem_assert_held(sem)	lockdep_assert_held(sem)



static inline void percpu_rwsem_release(struct percpu_rw_semaphore *sem,

					bool read, unsigned long ip)

{

	lock_release(&sem->rw_sem.dep_map, ip);

	lock_release(&sem->dep_map, ip);

#ifdef CONFIG_RWSEM_SPIN_ON_OWNER

	if (!read)

		atomic_long_set(&sem->rw_sem.owner, RWSEM_OWNER_UNKNOWN);
@@ -128,7 +137,7 @@ static inline void percpu_rwsem_release(struct percpu_rw_semaphore *sem, 
static inline void percpu_rwsem_acquire(struct percpu_rw_semaphore *sem,

					bool read, unsigned long ip)

{

	lock_acquire(&sem->rw_sem.dep_map, 0, 1, read, 1, NULL, ip);

	lock_acquire(&sem->dep_map, 0, 1, read, 1, NULL, ip);

#ifdef CONFIG_RWSEM_SPIN_ON_OWNER

	if (!read)

		atomic_long_set(&sem->rw_sem.owner, (long)current);

kernel/cpu.c

+2 −2
Original line number Diff line number Diff line
@@ -331,12 +331,12 @@ void lockdep_assert_cpus_held(void) 


static void lockdep_acquire_cpus_lock(void)

{

	rwsem_acquire(&cpu_hotplug_lock.rw_sem.dep_map, 0, 0, _THIS_IP_);

	rwsem_acquire(&cpu_hotplug_lock.dep_map, 0, 0, _THIS_IP_);

}



static void lockdep_release_cpus_lock(void)

{

	rwsem_release(&cpu_hotplug_lock.rw_sem.dep_map, _THIS_IP_);

	rwsem_release(&cpu_hotplug_lock.dep_map, _THIS_IP_);

}



/*

kernel/locking/percpu-rwsem.c

+12 −4
Original line number Diff line number Diff line
@@ -11,7 +11,7 @@ 
#include "rwsem.h"



int __percpu_init_rwsem(struct percpu_rw_semaphore *sem,

			const char *name, struct lock_class_key *rwsem_key)

			const char *name, struct lock_class_key *key)

{

	sem->read_count = alloc_percpu(int);

	if (unlikely(!sem->read_count))
@@ -19,9 +19,13 @@ int __percpu_init_rwsem(struct percpu_rw_semaphore *sem, 


	/* ->rw_sem represents the whole percpu_rw_semaphore for lockdep */

	rcu_sync_init(&sem->rss);

	__init_rwsem(&sem->rw_sem, name, rwsem_key);

	init_rwsem(&sem->rw_sem);

	rcuwait_init(&sem->writer);

	sem->readers_block = 0;

#ifdef CONFIG_DEBUG_LOCK_ALLOC

	debug_check_no_locks_freed((void *)sem, sizeof(*sem));

	lockdep_init_map(&sem->dep_map, name, key, 0);

#endif

	return 0;

}

EXPORT_SYMBOL_GPL(__percpu_init_rwsem);
@@ -142,10 +146,12 @@ static bool readers_active_check(struct percpu_rw_semaphore *sem) 


void percpu_down_write(struct percpu_rw_semaphore *sem)

{

	rwsem_acquire(&sem->dep_map, 0, 0, _RET_IP_);



	/* Notify readers to take the slow path. */

	rcu_sync_enter(&sem->rss);



	down_write(&sem->rw_sem);

	__down_write(&sem->rw_sem);



	/*

	 * Notify new readers to block; up until now, and thus throughout the
@@ -168,6 +174,8 @@ EXPORT_SYMBOL_GPL(percpu_down_write); 


void percpu_up_write(struct percpu_rw_semaphore *sem)

{

	rwsem_release(&sem->dep_map, _RET_IP_);



	/*

	 * Signal the writer is done, no fast path yet.

	 *
@@ -183,7 +191,7 @@ void percpu_up_write(struct percpu_rw_semaphore *sem) 
	/*

	 * Release the write lock, this will allow readers back in the game.

	 */

	up_write(&sem->rw_sem);

	__up_write(&sem->rw_sem);



	/*

	 * Once this completes (at least one RCU-sched grace period hence) the

kernel/locking/rwsem.c

+2 −2
Original line number Diff line number Diff line
@@ -1383,7 +1383,7 @@ static inline int __down_read_trylock(struct rw_semaphore *sem) 
/*

 * lock for writing

 */

static inline void __down_write(struct rw_semaphore *sem)

inline void __down_write(struct rw_semaphore *sem)

{

	long tmp = RWSEM_UNLOCKED_VALUE;
@@ -1446,7 +1446,7 @@ inline void __up_read(struct rw_semaphore *sem) 
/*

 * unlock after writing

 */

static inline void __up_write(struct rw_semaphore *sem)

inline void __up_write(struct rw_semaphore *sem)

{

	long tmp;

kernel/locking/rwsem.h

+2 −0
Original line number Diff line number Diff line
@@ -6,5 +6,7 @@ 


extern void __down_read(struct rw_semaphore *sem);

extern void __up_read(struct rw_semaphore *sem);

extern void __down_write(struct rw_semaphore *sem);

extern void __up_write(struct rw_semaphore *sem);



#endif /* __INTERNAL_RWSEM_H */

CRA Git | Maintained and supported by SUSTech CRA and CCSE