epoll: use the waitqueue lock to protect ep->wq

 *

 * 1) epmutex (mutex)

 * 2) ep->mtx (mutex)

 * 3) ep->lock (spinlock)

 * 3) ep->wq.lock (spinlock)

 *

 * The acquire order is the one listed above, from 1 to 3.

 * We need a spinlock (ep->lock) because we manipulate objects

 * We need a spinlock (ep->wq.lock) because we manipulate objects

 * from inside the poll callback, that might be triggered from

 * a wake_up() that in turn might be called from IRQ context.

 * So we can't sleep inside the poll callback and hence we need

 * of epoll file descriptors, we use the current recursion depth as

 * the lockdep subkey.

 * It is possible to drop the "ep->mtx" and to use the global

 * mutex "epmutex" (together with "ep->lock") to have it working,

 * mutex "epmutex" (together with "ep->wq.lock") to have it working,

 * but having "ep->mtx" will make the interface more scalable.

 * Events that require holding "epmutex" are very rare, while for

 * normal operations the epoll private "ep->mtx" will guarantee

 * This structure is stored inside the "private_data" member of the file

 * structure and represents the main data structure for the eventpoll

 * interface.

 *

 * Access to it is protected by the lock inside wq.

 */

struct eventpoll {

	/* Protect the access to this structure */

	spinlock_t lock;

	/*

	 * This mutex is used to ensure that files are not removed

	 * while epoll is using them. This is held during the event

	/*

	 * This is a single linked list that chains all the "struct epitem" that

	 * happened while transferring ready events to userspace w/out

	 * holding ->lock.

	 * holding ->wq.lock.

	 */

	struct epitem *ovflist;

	 * because we want the "sproc" callback to be able to do it

	 * in a lockless way.

	 */

	spin_lock_irqsave(&ep->lock, flags);

	spin_lock_irqsave(&ep->wq.lock, flags);

	list_splice_init(&ep->rdllist, &txlist);

	ep->ovflist = NULL;

	spin_unlock_irqrestore(&ep->lock, flags);

	spin_unlock_irqrestore(&ep->wq.lock, flags);

	/*

	 * Now call the callback function.

	 */

	res = (*sproc)(ep, &txlist, priv);

	spin_lock_irqsave(&ep->lock, flags);

	spin_lock_irqsave(&ep->wq.lock, flags);

	/*

	 * During the time we spent inside the "sproc" callback, some

	 * other events might have been queued by the poll callback.

		if (waitqueue_active(&ep->poll_wait))

			pwake++;

	}

	spin_unlock_irqrestore(&ep->lock, flags);

	spin_unlock_irqrestore(&ep->wq.lock, flags);

	if (!ep_locked)

		mutex_unlock(&ep->mtx);

	struct file *file = epi->ffd.file;

	/*

	 * Removes poll wait queue hooks. We _have_ to do this without holding

	 * the "ep->lock" otherwise a deadlock might occur. This because of the

	 * sequence of the lock acquisition. Here we do "ep->lock" then the wait

	 * queue head lock when unregistering the wait queue. The wakeup callback

	 * will run by holding the wait queue head lock and will call our callback

	 * that will try to get "ep->lock".

	 * Removes poll wait queue hooks.

	 */

	ep_unregister_pollwait(ep, epi);

	rb_erase_cached(&epi->rbn, &ep->rbr);

	spin_lock_irqsave(&ep->lock, flags);

	spin_lock_irqsave(&ep->wq.lock, flags);

	if (ep_is_linked(&epi->rdllink))

		list_del_init(&epi->rdllink);

	spin_unlock_irqrestore(&ep->lock, flags);

	spin_unlock_irqrestore(&ep->wq.lock, flags);

	wakeup_source_unregister(ep_wakeup_source(epi));

	/*

	 * Walks through the whole tree by freeing each "struct epitem". At this

	 * point we are sure no poll callbacks will be lingering around, and also by

	 * holding "epmutex" we can be sure that no file cleanup code will hit

	 * us during this operation. So we can avoid the lock on "ep->lock".

	 * us during this operation. So we can avoid the lock on "ep->wq.lock".

	 * We do not need to lock ep->mtx, either, we only do it to prevent

	 * a lockdep warning.

	 */

	if (unlikely(!ep))

		goto free_uid;

	spin_lock_init(&ep->lock);

	mutex_init(&ep->mtx);

	init_waitqueue_head(&ep->wq);

	init_waitqueue_head(&ep->poll_wait);

	__poll_t pollflags = key_to_poll(key);

	int ewake = 0;

	spin_lock_irqsave(&ep->lock, flags);

	spin_lock_irqsave(&ep->wq.lock, flags);

	ep_set_busy_poll_napi_id(epi);

		pwake++;

out_unlock:

	spin_unlock_irqrestore(&ep->lock, flags);

	spin_unlock_irqrestore(&ep->wq.lock, flags);

	/* We have to call this outside the lock */

	if (pwake)

		goto error_remove_epi;

	/* We have to drop the new item inside our item list to keep track of it */

	spin_lock_irqsave(&ep->lock, flags);

	spin_lock_irqsave(&ep->wq.lock, flags);

	/* record NAPI ID of new item if present */

	ep_set_busy_poll_napi_id(epi);

			pwake++;

	}

	spin_unlock_irqrestore(&ep->lock, flags);

	spin_unlock_irqrestore(&ep->wq.lock, flags);

	atomic_long_inc(&ep->user->epoll_watches);

	 * list, since that is used/cleaned only inside a section bound by "mtx".

	 * And ep_insert() is called with "mtx" held.

	 */

	spin_lock_irqsave(&ep->lock, flags);

	spin_lock_irqsave(&ep->wq.lock, flags);

	if (ep_is_linked(&epi->rdllink))

		list_del_init(&epi->rdllink);

	spin_unlock_irqrestore(&ep->lock, flags);

	spin_unlock_irqrestore(&ep->wq.lock, flags);

	wakeup_source_unregister(ep_wakeup_source(epi));

	 * 1) Flush epi changes above to other CPUs.  This ensures

	 *    we do not miss events from ep_poll_callback if an

	 *    event occurs immediately after we call f_op->poll().

	 *    We need this because we did not take ep->lock while

	 *    We need this because we did not take ep->wq.lock while

	 *    changing epi above (but ep_poll_callback does take

	 *    ep->lock).

	 *    ep->wq.lock).

	 *

	 * 2) We also need to ensure we do not miss _past_ events

	 *    when calling f_op->poll().  This barrier also

	 * list, push it inside.

	 */

	if (ep_item_poll(epi, &pt, 1)) {

		spin_lock_irq(&ep->lock);

		spin_lock_irq(&ep->wq.lock);

		if (!ep_is_linked(&epi->rdllink)) {

			list_add_tail(&epi->rdllink, &ep->rdllist);

			ep_pm_stay_awake(epi);

			if (waitqueue_active(&ep->poll_wait))

				pwake++;

		}

		spin_unlock_irq(&ep->lock);

		spin_unlock_irq(&ep->wq.lock);

	}

	/* We have to call this outside the lock */

		 * caller specified a non blocking operation.

		 */

		timed_out = 1;

		spin_lock_irqsave(&ep->lock, flags);

		spin_lock_irqsave(&ep->wq.lock, flags);

		goto check_events;

	}

	if (!ep_events_available(ep))

		ep_busy_loop(ep, timed_out);

	spin_lock_irqsave(&ep->lock, flags);

	spin_lock_irqsave(&ep->wq.lock, flags);

	if (!ep_events_available(ep)) {

		/*

				break;

			}

			spin_unlock_irqrestore(&ep->lock, flags);

			spin_unlock_irqrestore(&ep->wq.lock, flags);

			if (!schedule_hrtimeout_range(to, slack, HRTIMER_MODE_ABS))

				timed_out = 1;

			spin_lock_irqsave(&ep->lock, flags);

			spin_lock_irqsave(&ep->wq.lock, flags);

		}

		__remove_wait_queue(&ep->wq, &wait);

	/* Is it worth to try to dig for events ? */

	eavail = ep_events_available(ep);

	spin_unlock_irqrestore(&ep->lock, flags);

	spin_unlock_irqrestore(&ep->wq.lock, flags);

	/*

	 * Try to transfer events to user space. In case we get 0 events and