Commit 5293c2ef authored by Peter Zijlstra's avatar Peter Zijlstra Committed by Thomas Gleixner
Browse files

futex,rt_mutex: Provide futex specific rt_mutex API 



Part of what makes futex_unlock_pi() intricate is that
rt_mutex_futex_unlock() -> rt_mutex_slowunlock() can drop
rt_mutex::wait_lock.

This means it cannot rely on the atomicy of wait_lock, which would be
preferred in order to not rely on hb->lock so much.

The reason rt_mutex_slowunlock() needs to drop wait_lock is because it can
race with the rt_mutex fastpath, however futexes have their own fast path.

Since futexes already have a bunch of separate rt_mutex accessors, complete
that set and implement a rt_mutex variant without fastpath for them.

Signed-off-by: default avatarPeter Zijlstra (Intel) <peterz@infradead.org>
Cc: juri.lelli@arm.com
Cc: bigeasy@linutronix.de
Cc: xlpang@redhat.com
Cc: rostedt@goodmis.org
Cc: mathieu.desnoyers@efficios.com
Cc: jdesfossez@efficios.com
Cc: dvhart@infradead.org
Cc: bristot@redhat.com
Link: http://lkml.kernel.org/r/20170322104151.702962446@infradead.org


Signed-off-by: default avatarThomas Gleixner <tglx@linutronix.de>
parent fffa954f

kernel/futex.c

+14 −16
Original line number Diff line number Diff line
@@ -916,7 +916,7 @@ void exit_pi_state_list(struct task_struct *curr) 
		pi_state->owner = NULL;

		raw_spin_unlock_irq(&curr->pi_lock);



		rt_mutex_unlock(&pi_state->pi_mutex);

		rt_mutex_futex_unlock(&pi_state->pi_mutex);



		spin_unlock(&hb->lock);
@@ -1364,20 +1364,18 @@ static int wake_futex_pi(u32 __user *uaddr, u32 uval, struct futex_q *top_waiter 
	pi_state->owner = new_owner;

	raw_spin_unlock(&new_owner->pi_lock);



	raw_spin_unlock_irq(&pi_state->pi_mutex.wait_lock);



	deboost = rt_mutex_futex_unlock(&pi_state->pi_mutex, &wake_q);



	/*

	 * First unlock HB so the waiter does not spin on it once he got woken

	 * up. Second wake up the waiter before the priority is adjusted. If we

	 * deboost first (and lose our higher priority), then the task might get

	 * scheduled away before the wake up can take place.

	 * We've updated the uservalue, this unlock cannot fail.

	 */

	deboost = __rt_mutex_futex_unlock(&pi_state->pi_mutex, &wake_q);



	raw_spin_unlock_irq(&pi_state->pi_mutex.wait_lock);

	spin_unlock(&hb->lock);



	if (deboost) {

		wake_up_q(&wake_q);

	if (deboost)

		rt_mutex_adjust_prio(current);

	}



	return 0;

}
@@ -2253,7 +2251,7 @@ static int fixup_owner(u32 __user *uaddr, struct futex_q *q, int locked) 
		 * task acquired the rt_mutex after we removed ourself from the

		 * rt_mutex waiters list.

		 */

		if (rt_mutex_trylock(&q->pi_state->pi_mutex)) {

		if (rt_mutex_futex_trylock(&q->pi_state->pi_mutex)) {

			locked = 1;

			goto out;

		}
@@ -2568,7 +2566,7 @@ retry_private: 
	if (!trylock) {

		ret = rt_mutex_timed_futex_lock(&q.pi_state->pi_mutex, to);

	} else {

		ret = rt_mutex_trylock(&q.pi_state->pi_mutex);

		ret = rt_mutex_futex_trylock(&q.pi_state->pi_mutex);

		/* Fixup the trylock return value: */

		ret = ret ? 0 : -EWOULDBLOCK;

	}
@@ -2591,7 +2589,7 @@ retry_private: 
	 * it and return the fault to userspace.

	 */

	if (ret && (rt_mutex_owner(&q.pi_state->pi_mutex) == current))

		rt_mutex_unlock(&q.pi_state->pi_mutex);

		rt_mutex_futex_unlock(&q.pi_state->pi_mutex);



	/* Unqueue and drop the lock */

	unqueue_me_pi(&q);
@@ -2898,7 +2896,7 @@ static int futex_wait_requeue_pi(u32 __user *uaddr, unsigned int flags, 
			spin_lock(q.lock_ptr);

			ret = fixup_pi_state_owner(uaddr2, &q, current);

			if (ret && rt_mutex_owner(&q.pi_state->pi_mutex) == current)

				rt_mutex_unlock(&q.pi_state->pi_mutex);

				rt_mutex_futex_unlock(&q.pi_state->pi_mutex);

			/*

			 * Drop the reference to the pi state which

			 * the requeue_pi() code acquired for us.
@@ -2938,7 +2936,7 @@ static int futex_wait_requeue_pi(u32 __user *uaddr, unsigned int flags, 
		 * userspace.

		 */

		if (ret && rt_mutex_owner(pi_mutex) == current)

			rt_mutex_unlock(pi_mutex);

			rt_mutex_futex_unlock(pi_mutex);



		/* Unqueue and drop the lock. */

		unqueue_me_pi(&q);

kernel/locking/rtmutex.c

+41 −14
Original line number Diff line number Diff line
@@ -1488,15 +1488,23 @@ EXPORT_SYMBOL_GPL(rt_mutex_lock_interruptible); 


/*

 * Futex variant with full deadlock detection.

 * Futex variants must not use the fast-path, see __rt_mutex_futex_unlock().

 */

int rt_mutex_timed_futex_lock(struct rt_mutex *lock,

int __sched rt_mutex_timed_futex_lock(struct rt_mutex *lock,

			      struct hrtimer_sleeper *timeout)

{

	might_sleep();



	return rt_mutex_timed_fastlock(lock, TASK_INTERRUPTIBLE, timeout,

				       RT_MUTEX_FULL_CHAINWALK,

				       rt_mutex_slowlock);

	return rt_mutex_slowlock(lock, TASK_INTERRUPTIBLE,

				 timeout, RT_MUTEX_FULL_CHAINWALK);

}



/*

 * Futex variant, must not use fastpath.

 */

int __sched rt_mutex_futex_trylock(struct rt_mutex *lock)

{

	return rt_mutex_slowtrylock(lock);

}



/**
@@ -1555,19 +1563,38 @@ void __sched rt_mutex_unlock(struct rt_mutex *lock) 
EXPORT_SYMBOL_GPL(rt_mutex_unlock);



/**

 * rt_mutex_futex_unlock - Futex variant of rt_mutex_unlock

 * @lock: the rt_mutex to be unlocked

 *

 * Returns: true/false indicating whether priority adjustment is

 * required or not.

 * Futex variant, that since futex variants do not use the fast-path, can be

 * simple and will not need to retry.

 */

bool __sched rt_mutex_futex_unlock(struct rt_mutex *lock,

				   struct wake_q_head *wqh)

bool __sched __rt_mutex_futex_unlock(struct rt_mutex *lock,

				    struct wake_q_head *wake_q)

{

	if (likely(rt_mutex_cmpxchg_release(lock, current, NULL)))

		return false;

	lockdep_assert_held(&lock->wait_lock);



	debug_rt_mutex_unlock(lock);



	if (!rt_mutex_has_waiters(lock)) {

		lock->owner = NULL;

		return false; /* done */

	}



	mark_wakeup_next_waiter(wake_q, lock);

	return true; /* deboost and wakeups */

}



	return rt_mutex_slowunlock(lock, wqh);

void __sched rt_mutex_futex_unlock(struct rt_mutex *lock)

{

	DEFINE_WAKE_Q(wake_q);

	bool deboost;



	raw_spin_lock_irq(&lock->wait_lock);

	deboost = __rt_mutex_futex_unlock(lock, &wake_q);

	raw_spin_unlock_irq(&lock->wait_lock);



	if (deboost) {

		wake_up_q(&wake_q);

		rt_mutex_adjust_prio(current);

	}

}



/**

kernel/locking/rtmutex_common.h

+7 −2
Original line number Diff line number Diff line
@@ -109,9 +109,14 @@ extern int rt_mutex_start_proxy_lock(struct rt_mutex *lock, 
extern int rt_mutex_finish_proxy_lock(struct rt_mutex *lock,

				      struct hrtimer_sleeper *to,

				      struct rt_mutex_waiter *waiter);



extern int rt_mutex_timed_futex_lock(struct rt_mutex *l, struct hrtimer_sleeper *to);

extern bool rt_mutex_futex_unlock(struct rt_mutex *lock,

extern int rt_mutex_futex_trylock(struct rt_mutex *l);



extern void rt_mutex_futex_unlock(struct rt_mutex *lock);

extern bool __rt_mutex_futex_unlock(struct rt_mutex *lock,

				 struct wake_q_head *wqh);



extern void rt_mutex_adjust_prio(struct task_struct *task);



#ifdef CONFIG_DEBUG_RT_MUTEXES

CRA Git | Maintained and supported by SUSTech CRA and CCSE