locking/rwsem: Add reader-owned state to the owner field (19c5d690) · Commits · 戴 / test

kernel/locking/rwsem-xadd.c

+22 −19

Original line number	Diff line number	Diff line
		@@ -163,6 +163,12 @@ __rwsem_mark_wake(struct rw_semaphore *sem,
		/* Last active locker left. Retry waking readers. */
		goto try_reader_grant;
		}
		/*
		* It is not really necessary to set it to reader-owned here,
		* but it gives the spinners an early indication that the
		* readers now have the lock.
		*/
		rwsem_set_reader_owned(sem);
		}

		/* Grant an infinite number of read locks to the readers at the front
		@@ -325,16 +331,11 @@ static inline bool rwsem_can_spin_on_owner(struct rw_semaphore *sem)

		rcu_read_lock();
		owner = READ_ONCE(sem->owner);
		if (!owner) {
		long count = atomic_long_read(&sem->count);
		if (!rwsem_owner_is_writer(owner)) {
		/*
		* If sem->owner is not set, yet we have just recently entered the
		* slowpath with the lock being active, then there is a possibility
		* reader(s) may have the lock. To be safe, bail spinning in these
		* situations.
		* Don't spin if the rwsem is readers owned.
		*/
		if (count & RWSEM_ACTIVE_MASK)
		ret = false;
		ret = !rwsem_owner_is_reader(owner);
		goto done;
		}

		@@ -347,8 +348,6 @@ done:
		static noinline
		bool rwsem_spin_on_owner(struct rw_semaphore sem, struct task_struct owner)
		{
		long count;

		rcu_read_lock();
		while (sem->owner == owner) {
		/*
		@@ -369,16 +368,11 @@ bool rwsem_spin_on_owner(struct rw_semaphore sem, struct task_struct owner)
		}
		rcu_read_unlock();

		if (READ_ONCE(sem->owner))
		return true; /* new owner, continue spinning */

		/*
		* When the owner is not set, the lock could be free or
		* held by readers. Check the counter to verify the
		* state.
		* If there is a new owner or the owner is not set, we continue
		* spinning.
		*/
		count = atomic_long_read(&sem->count);
		return (count == 0 \|\| count == RWSEM_WAITING_BIAS);
		return !rwsem_owner_is_reader(READ_ONCE(sem->owner));
		}

		static bool rwsem_optimistic_spin(struct rw_semaphore *sem)
		@@ -397,7 +391,16 @@ static bool rwsem_optimistic_spin(struct rw_semaphore *sem)

		while (true) {
		owner = READ_ONCE(sem->owner);
		if (owner && !rwsem_spin_on_owner(sem, owner))
		/*
		* Don't spin if
		* 1) the owner is a reader as we we can't determine if the
		* reader is actively running or not.
		* 2) The rwsem_spin_on_owner() returns false which means
		* the owner isn't running.
		*/
		if (rwsem_owner_is_reader(owner) \|\|
		(rwsem_owner_is_writer(owner) &&
		!rwsem_spin_on_owner(sem, owner)))
		break;

		/* wait_lock will be acquired if write_lock is obtained */

kernel/locking/rwsem.c

+6 −2

Original line number	Diff line number	Diff line
		@@ -22,6 +22,7 @@ void __sched down_read(struct rw_semaphore *sem)
		rwsem_acquire_read(&sem->dep_map, 0, 0, _RET_IP_);

		LOCK_CONTENDED(sem, __down_read_trylock, __down_read);
		rwsem_set_reader_owned(sem);
		}

		EXPORT_SYMBOL(down_read);
		@@ -33,8 +34,10 @@ int down_read_trylock(struct rw_semaphore *sem)
		{
		int ret = __down_read_trylock(sem);

		if (ret == 1)
		if (ret == 1) {
		rwsem_acquire_read(&sem->dep_map, 0, 1, _RET_IP_);
		rwsem_set_reader_owned(sem);
		}
		return ret;
		}

		@@ -124,7 +127,7 @@ void downgrade_write(struct rw_semaphore *sem)
		* lockdep: a downgraded write will live on as a write
		* dependency.
		*/
		rwsem_clear_owner(sem);
		rwsem_set_reader_owned(sem);
		__downgrade_write(sem);
		}

		@@ -138,6 +141,7 @@ void down_read_nested(struct rw_semaphore *sem, int subclass)
		rwsem_acquire_read(&sem->dep_map, subclass, 0, _RET_IP_);

		LOCK_CONTENDED(sem, __down_read_trylock, __down_read);
		rwsem_set_reader_owned(sem);
		}

		EXPORT_SYMBOL(down_read_nested);

kernel/locking/rwsem.h

+41 −0

Original line number	Diff line number	Diff line
		/*
		* The owner field of the rw_semaphore structure will be set to
		* RWSEM_READ_OWNED when a reader grabs the lock. A writer will clear
		* the owner field when it unlocks. A reader, on the other hand, will
		* not touch the owner field when it unlocks.
		*
		* In essence, the owner field now has the following 3 states:
		* 1) 0
		* - lock is free or the owner hasn't set the field yet
		* 2) RWSEM_READER_OWNED
		* - lock is currently or previously owned by readers (lock is free
		* or not set by owner yet)
		* 3) Other non-zero value
		* - a writer owns the lock
		*/
		#define RWSEM_READER_OWNED ((struct task_struct *)1UL)

		#ifdef CONFIG_RWSEM_SPIN_ON_OWNER
		static inline void rwsem_set_owner(struct rw_semaphore *sem)
		{
		@@ -9,6 +26,26 @@ static inline void rwsem_clear_owner(struct rw_semaphore *sem)
		sem->owner = NULL;
		}

		static inline void rwsem_set_reader_owned(struct rw_semaphore *sem)
		{
		/*
		* We check the owner value first to make sure that we will only
		* do a write to the rwsem cacheline when it is really necessary
		* to minimize cacheline contention.
		*/
		if (sem->owner != RWSEM_READER_OWNED)
		sem->owner = RWSEM_READER_OWNED;
		}

		static inline bool rwsem_owner_is_writer(struct task_struct *owner)
		{
		return owner && owner != RWSEM_READER_OWNED;
		}

		static inline bool rwsem_owner_is_reader(struct task_struct *owner)
		{
		return owner == RWSEM_READER_OWNED;
		}
		#else
		static inline void rwsem_set_owner(struct rw_semaphore *sem)
		{
		@@ -17,4 +54,8 @@ static inline void rwsem_set_owner(struct rw_semaphore *sem)
		static inline void rwsem_clear_owner(struct rw_semaphore *sem)
		{
		}

		static inline void rwsem_set_reader_owned(struct rw_semaphore *sem)
		{
		}
		#endif

Admin message