Commit 1a52084d authored by Darren Hart's avatar Darren Hart Committed by Thomas Gleixner
Browse files

futex: split out atomic logic from futex_lock_pi() 



Refactor the atomic portion of futex_lock_pi() into futex_lock_pi_atomic().

This logic will be needed by requeue_pi, so modularize it to reduce
code duplication.  The only significant change is passing of the task
to try and take the lock for.  This simplifies the -EDEADLK test as if
the lock is owned by task t, it's a deadlock, regardless of if we are
doing requeue pi or not.  This patch updates the corresponding comment
accordingly.

Signed-off-by: default avatarDarren Hart <dvhltc@us.ibm.com>
Reviewed-by: default avatarThomas Gleixner <tglx@linutronix.de>
Signed-off-by: default avatarThomas Gleixner <tglx@linutronix.de>
parent 4b1c486b

kernel/futex.c

+130 −94
Original line number Diff line number Diff line
@@ -556,6 +556,127 @@ lookup_pi_state(u32 uval, struct futex_hash_bucket *hb, 
	return 0;

}



/**

 * futex_lock_pi_atomic() - atomic work required to acquire a pi aware futex

 * @uaddr:	the pi futex user address

 * @hb:		the pi futex hash bucket

 * @key:	the futex key associated with uaddr and hb

 * @ps:		the pi_state pointer where we store the result of the lookup

 * @task:	the task to perform the atomic lock work for.  This will be

 * 		"current" except in the case of requeue pi.

 *

 * Returns:

 *  0 - ready to wait

 *  1 - acquired the lock

 * <0 - error

 *

 * The hb->lock and futex_key refs shall be held by the caller.

 */

static int futex_lock_pi_atomic(u32 __user *uaddr, struct futex_hash_bucket *hb,

				union futex_key *key,

				struct futex_pi_state **ps,

				struct task_struct *task)

{

	int lock_taken, ret, ownerdied = 0;

	u32 uval, newval, curval;



retry:

	ret = lock_taken = 0;



	/*

	 * To avoid races, we attempt to take the lock here again

	 * (by doing a 0 -> TID atomic cmpxchg), while holding all

	 * the locks. It will most likely not succeed.

	 */

	newval = task_pid_vnr(task);



	curval = cmpxchg_futex_value_locked(uaddr, 0, newval);



	if (unlikely(curval == -EFAULT))

		return -EFAULT;



	/*

	 * Detect deadlocks.

	 */

	if ((unlikely((curval & FUTEX_TID_MASK) == task_pid_vnr(task))))

		return -EDEADLK;



	/*

	 * Surprise - we got the lock. Just return to userspace:

	 */

	if (unlikely(!curval))

		return 1;



	uval = curval;



	/*

	 * Set the FUTEX_WAITERS flag, so the owner will know it has someone

	 * to wake at the next unlock.

	 */

	newval = curval | FUTEX_WAITERS;



	/*

	 * There are two cases, where a futex might have no owner (the

	 * owner TID is 0): OWNER_DIED. We take over the futex in this

	 * case. We also do an unconditional take over, when the owner

	 * of the futex died.

	 *

	 * This is safe as we are protected by the hash bucket lock !

	 */

	if (unlikely(ownerdied || !(curval & FUTEX_TID_MASK))) {

		/* Keep the OWNER_DIED bit */

		newval = (curval & ~FUTEX_TID_MASK) | task_pid_vnr(task);

		ownerdied = 0;

		lock_taken = 1;

	}



	curval = cmpxchg_futex_value_locked(uaddr, uval, newval);



	if (unlikely(curval == -EFAULT))

		return -EFAULT;

	if (unlikely(curval != uval))

		goto retry;



	/*

	 * We took the lock due to owner died take over.

	 */

	if (unlikely(lock_taken))

		return 1;



	/*

	 * We dont have the lock. Look up the PI state (or create it if

	 * we are the first waiter):

	 */

	ret = lookup_pi_state(uval, hb, key, ps);



	if (unlikely(ret)) {

		switch (ret) {

		case -ESRCH:

			/*

			 * No owner found for this futex. Check if the

			 * OWNER_DIED bit is set to figure out whether

			 * this is a robust futex or not.

			 */

			if (get_futex_value_locked(&curval, uaddr))

				return -EFAULT;



			/*

			 * We simply start over in case of a robust

			 * futex. The code above will take the futex

			 * and return happy.

			 */

			if (curval & FUTEX_OWNER_DIED) {

				ownerdied = 1;

				goto retry;

			}

		default:

			break;

		}

	}



	return ret;

}



/*

 * The hash bucket lock must be held when this is called.

 * Afterwards, the futex_q must not be accessed.
@@ -1340,9 +1461,9 @@ static int futex_lock_pi(u32 __user *uaddr, int fshared, 
	struct hrtimer_sleeper timeout, *to = NULL;

	struct task_struct *curr = current;

	struct futex_hash_bucket *hb;

	u32 uval, newval, curval;

	u32 uval;

	struct futex_q q;

	int ret, lock_taken, ownerdied = 0;

	int ret;



	if (refill_pi_state_cache())

		return -ENOMEM;
@@ -1365,81 +1486,15 @@ retry: 
retry_private:

	hb = queue_lock(&q);



retry_locked:

	ret = lock_taken = 0;



	/*

	 * To avoid races, we attempt to take the lock here again

	 * (by doing a 0 -> TID atomic cmpxchg), while holding all

	 * the locks. It will most likely not succeed.

	 */

	newval = task_pid_vnr(current);



	curval = cmpxchg_futex_value_locked(uaddr, 0, newval);



	if (unlikely(curval == -EFAULT))

		goto uaddr_faulted;



	/*

	 * Detect deadlocks. In case of REQUEUE_PI this is a valid

	 * situation and we return success to user space.

	 */

	if (unlikely((curval & FUTEX_TID_MASK) == task_pid_vnr(current))) {

		ret = -EDEADLK;

		goto out_unlock_put_key;

	}



	/*

	 * Surprise - we got the lock. Just return to userspace:

	 */

	if (unlikely(!curval))

		goto out_unlock_put_key;



	uval = curval;



	/*

	 * Set the WAITERS flag, so the owner will know it has someone

	 * to wake at next unlock

	 */

	newval = curval | FUTEX_WAITERS;



	/*

	 * There are two cases, where a futex might have no owner (the

	 * owner TID is 0): OWNER_DIED. We take over the futex in this

	 * case. We also do an unconditional take over, when the owner

	 * of the futex died.

	 *

	 * This is safe as we are protected by the hash bucket lock !

	 */

	if (unlikely(ownerdied || !(curval & FUTEX_TID_MASK))) {

		/* Keep the OWNER_DIED bit */

		newval = (curval & ~FUTEX_TID_MASK) | task_pid_vnr(current);

		ownerdied = 0;

		lock_taken = 1;

	}



	curval = cmpxchg_futex_value_locked(uaddr, uval, newval);



	if (unlikely(curval == -EFAULT))

		goto uaddr_faulted;

	if (unlikely(curval != uval))

		goto retry_locked;



	/*

	 * We took the lock due to owner died take over.

	 */

	if (unlikely(lock_taken))

		goto out_unlock_put_key;



	/*

	 * We dont have the lock. Look up the PI state (or create it if

	 * we are the first waiter):

	 */

	ret = lookup_pi_state(uval, hb, &q.key, &q.pi_state);



	ret = futex_lock_pi_atomic(uaddr, hb, &q.key, &q.pi_state, current);

	if (unlikely(ret)) {

		switch (ret) {



		case 1:

			/* We got the lock. */

			ret = 0;

			goto out_unlock_put_key;

		case -EFAULT:

			goto uaddr_faulted;

		case -EAGAIN:

			/*

			 * Task is exiting and we just wait for the
@@ -1449,25 +1504,6 @@ retry_locked: 
			put_futex_key(fshared, &q.key);

			cond_resched();

			goto retry;



		case -ESRCH:

			/*

			 * No owner found for this futex. Check if the

			 * OWNER_DIED bit is set to figure out whether

			 * this is a robust futex or not.

			 */

			if (get_futex_value_locked(&curval, uaddr))

				goto uaddr_faulted;



			/*

			 * We simply start over in case of a robust

			 * futex. The code above will take the futex

			 * and return happy.

			 */

			if (curval & FUTEX_OWNER_DIED) {

				ownerdied = 1;

				goto retry_locked;

			}

		default:

			goto out_unlock_put_key;

		}

CRA Git | Maintained and supported by SUSTech CRA and CCSE