futex: Simplify futex_lock_pi_atomic() and make it more robust (af54d6a1) · Commits · 戴 / test

kernel/futex.c

+61 −87

Original line number	Diff line number	Diff line
		@@ -956,6 +956,17 @@ static int lookup_pi_state(u32 uval, struct futex_hash_bucket *hb,
		return attach_to_pi_owner(uval, key, ps);
		}

		static int lock_pi_update_atomic(u32 __user *uaddr, u32 uval, u32 newval)
		{
		u32 uninitialized_var(curval);

		if (unlikely(cmpxchg_futex_value_locked(&curval, uaddr, uval, newval)))
		return -EFAULT;

		/If user space value changed, let the caller retry /
		return curval != uval ? -EAGAIN : 0;
		}

		/**
		* futex_lock_pi_atomic() - Atomic work required to acquire a pi aware futex
		* @uaddr: the pi futex user address
		@@ -979,113 +990,69 @@ static int futex_lock_pi_atomic(u32 __user uaddr, struct futex_hash_bucket hb,
		struct futex_pi_state **ps,
		struct task_struct *task, int set_waiters)
		{
		int lock_taken, ret, force_take = 0;
		u32 uval, newval, curval, vpid = task_pid_vnr(task);

		retry:
		ret = lock_taken = 0;
		u32 uval, newval, vpid = task_pid_vnr(task);
		struct futex_q *match;
		int ret;

		/*
		* To avoid races, we attempt to take the lock here again
		* (by doing a 0 -> TID atomic cmpxchg), while holding all
		* the locks. It will most likely not succeed.
		* Read the user space value first so we can validate a few
		* things before proceeding further.
		*/
		newval = vpid;
		if (set_waiters)
		newval \|= FUTEX_WAITERS;

		if (unlikely(cmpxchg_futex_value_locked(&curval, uaddr, 0, newval)))
		if (get_futex_value_locked(&uval, uaddr))
		return -EFAULT;

		/*
		* Detect deadlocks.
		*/
		if ((unlikely((curval & FUTEX_TID_MASK) == vpid)))
		if ((unlikely((uval & FUTEX_TID_MASK) == vpid)))
		return -EDEADLK;

		/*
		* Surprise - we got the lock, but we do not trust user space at all.
		* Lookup existing state first. If it exists, try to attach to
		* its pi_state.
		*/
		if (unlikely(!curval)) {
		/*
		* We verify whether there is kernel state for this
		* futex. If not, we can safely assume, that the 0 ->
		* TID transition is correct. If state exists, we do
		* not bother to fixup the user space state as it was
		* corrupted already.
		*/
		return futex_top_waiter(hb, key) ? -EINVAL : 1;
		}

		uval = curval;

		/*
		* Set the FUTEX_WAITERS flag, so the owner will know it has someone
		* to wake at the next unlock.
		*/
		newval = curval \| FUTEX_WAITERS;
		match = futex_top_waiter(hb, key);
		if (match)
		return attach_to_pi_state(uval, match->pi_state, ps);

		/*
		* Should we force take the futex? See below.
		* No waiter and user TID is 0. We are here because the
		* waiters or the owner died bit is set or called from
		* requeue_cmp_pi or for whatever reason something took the
		* syscall.
		*/
		if (unlikely(force_take)) {
		if (!(uval & FUTEX_TID_MASK)) {
		/*
		* Keep the OWNER_DIED and the WAITERS bit and set the
		* new TID value.
		* We take over the futex. No other waiters and the user space
		* TID is 0. We preserve the owner died bit.
		*/
		newval = (curval & ~FUTEX_TID_MASK) \| vpid;
		force_take = 0;
		lock_taken = 1;
		}

		if (unlikely(cmpxchg_futex_value_locked(&curval, uaddr, uval, newval)))
		return -EFAULT;
		if (unlikely(curval != uval))
		goto retry;
		newval = uval & FUTEX_OWNER_DIED;
		newval \|= vpid;

		/*
		* We took the lock due to forced take over.
		*/
		if (unlikely(lock_taken))
		return 1;
		/* The futex requeue_pi code can enforce the waiters bit */
		if (set_waiters)
		newval \|= FUTEX_WAITERS;

		/*
		* We dont have the lock. Look up the PI state (or create it if
		* we are the first waiter):
		*/
		ret = lookup_pi_state(uval, hb, key, ps);
		ret = lock_pi_update_atomic(uaddr, uval, newval);
		/* If the take over worked, return 1 */
		return ret < 0 ? ret : 1;
		}

		if (unlikely(ret)) {
		switch (ret) {
		case -ESRCH:
		/*
		* We failed to find an owner for this
		* futex. So we have no pi_state to block
		* on. This can happen in two cases:
		*
		* 1) The owner died
		* 2) A stale FUTEX_WAITERS bit
		*
		* Re-read the futex value.
		* First waiter. Set the waiters bit before attaching ourself to
		* the owner. If owner tries to unlock, it will be forced into
		* the kernel and blocked on hb->lock.
		*/
		if (get_futex_value_locked(&curval, uaddr))
		return -EFAULT;

		newval = uval \| FUTEX_WAITERS;
		ret = lock_pi_update_atomic(uaddr, uval, newval);
		if (ret)
		return ret;
		/*
		* If the owner died or we have a stale
		* WAITERS bit the owner TID in the user space
		* futex is 0.
		* If the update of the user space value succeeded, we try to
		* attach to the owner. If that fails, no harm done, we only
		* set the FUTEX_WAITERS bit in the user space variable.
		*/
		if (!(curval & FUTEX_TID_MASK)) {
		force_take = 1;
		goto retry;
		}
		default:
		break;
		}
		}

		return ret;
		return attach_to_pi_owner(uval, key, ps);
		}

		/**
		@@ -1659,7 +1626,12 @@ retry_private:
		goto retry;
		goto out;
		case -EAGAIN:
		/* The owner was exiting, try again. */
		/*
		* Two reasons for this:
		* - Owner is exiting and we just wait for the
		* exit to complete.
		* - The user space value changed.
		*/
		double_unlock_hb(hb1, hb2);
		hb_waiters_dec(hb2);
		put_futex_key(&key2);
		@@ -2316,8 +2288,10 @@ retry_private:
		goto uaddr_faulted;
		case -EAGAIN:
		/*
		* Task is exiting and we just wait for the
		* Two reasons for this:
		* - Task is exiting and we just wait for the
		* exit to complete.
		* - The user space value changed.
		*/
		queue_unlock(hb);
		put_futex_key(&q.key);

Admin message