seqlock: seqcount_LOCKNAME_t: Introduce PREEMPT_RT support

#include <linux/kcsan-checks.h>

#include <linux/lockdep.h>

#include <linux/mutex.h>

#include <linux/ww_mutex.h>

#include <linux/preempt.h>

#include <linux/spinlock.h>

 * See Documentation/locking/seqlock.rst

 */

#ifdef CONFIG_LOCKDEP

/*

 * For PREEMPT_RT, seqcount_LOCKNAME_t write side critical sections cannot

 * disable preemption. It can lead to higher latencies, and the write side

 * sections will not be able to acquire locks which become sleeping locks

 * (e.g. spinlock_t).

 *

 * To remain preemptible while avoiding a possible livelock caused by the

 * reader preempting the writer, use a different technique: let the reader

 * detect if a seqcount_LOCKNAME_t writer is in progress. If that is the

 * case, acquire then release the associated LOCKNAME writer serialization

 * lock. This will allow any possibly-preempted writer to make progress

 * until the end of its writer serialization lock critical section.

 *

 * This lock-unlock technique must be implemented for all of PREEMPT_RT

 * sleeping locks.  See Documentation/locking/locktypes.rst

 */

#if defined(CONFIG_LOCKDEP) || defined(CONFIG_PREEMPT_RT)

#define __SEQ_LOCK(expr)	expr

#else

#define __SEQ_LOCK(expr)

 *

 * @lockname:		"LOCKNAME" part of seqcount_LOCKNAME_t

 * @locktype:		LOCKNAME canonical C data type

 * @preemptible:	preemptibility of above lockname

 * @preemptible:	preemptibility of above locktype

 * @lockmember:		argument for lockdep_assert_held()

 * @lockbase:		associated lock release function (prefix only)

 * @lock_acquire:	associated lock acquisition function (full call)

 */

#define SEQCOUNT_LOCKNAME(lockname, locktype, preemptible, lockmember)	\

#define SEQCOUNT_LOCKNAME(lockname, locktype, preemptible, lockmember, lockbase, lock_acquire) \

typedef struct seqcount_##lockname {					\

	seqcount_t		seqcount;				\

	__SEQ_LOCK(locktype	*lock);					\

static __always_inline unsigned						\

__seqprop_##lockname##_sequence(const seqcount_##lockname##_t *s)	\

{									\

	return READ_ONCE(s->seqcount.sequence);				\

	unsigned seq = READ_ONCE(s->seqcount.sequence);			\

									\

	if (!IS_ENABLED(CONFIG_PREEMPT_RT))				\

		return seq;						\

									\

	if (preemptible && unlikely(seq & 1)) {				\

		__SEQ_LOCK(lock_acquire);				\

		__SEQ_LOCK(lockbase##_unlock(s->lock));			\

									\

		/*							\

		 * Re-read the sequence counter since the (possibly	\

		 * preempted) writer made progress.			\

		 */							\

		seq = READ_ONCE(s->seqcount.sequence);			\

	}								\

									\

	return seq;							\

}									\

									\

static __always_inline bool						\

__seqprop_##lockname##_preemptible(const seqcount_##lockname##_t *s)	\

{									\

	if (!IS_ENABLED(CONFIG_PREEMPT_RT))				\

		return preemptible;					\

									\

	/* PREEMPT_RT relies on the above LOCK+UNLOCK */		\

	return false;							\

}									\

									\

static __always_inline void						\

	lockdep_assert_preemption_disabled();

}

SEQCOUNT_LOCKNAME(raw_spinlock,	raw_spinlock_t,		false,	s->lock)

SEQCOUNT_LOCKNAME(spinlock,	spinlock_t,		false,	s->lock)

SEQCOUNT_LOCKNAME(rwlock,	rwlock_t,		false,	s->lock)

SEQCOUNT_LOCKNAME(mutex,	struct mutex,		true,	s->lock)

SEQCOUNT_LOCKNAME(ww_mutex,	struct ww_mutex,	true,	&s->lock->base)

#define __SEQ_RT	IS_ENABLED(CONFIG_PREEMPT_RT)

SEQCOUNT_LOCKNAME(raw_spinlock, raw_spinlock_t,  false,    s->lock,        raw_spin, raw_spin_lock(s->lock))

SEQCOUNT_LOCKNAME(spinlock,     spinlock_t,      __SEQ_RT, s->lock,        spin,     spin_lock(s->lock))

SEQCOUNT_LOCKNAME(rwlock,       rwlock_t,        __SEQ_RT, s->lock,        read,     read_lock(s->lock))

SEQCOUNT_LOCKNAME(mutex,        struct mutex,    true,     s->lock,        mutex,    mutex_lock(s->lock))

SEQCOUNT_LOCKNAME(ww_mutex,     struct ww_mutex, true,     &s->lock->base, ww_mutex, ww_mutex_lock(s->lock, NULL))

/*

 * SEQCNT_LOCKNAME_ZERO - static initializer for seqcount_LOCKNAME_t