Commit 1493c47f authored by Soheil Hassas Yeganeh's avatar Soheil Hassas Yeganeh Committed by Linus Torvalds
Browse files

epoll: simplify and optimize busy loop logic 

ep_events_available() is called multiple times around the busy loop logic,
even though the logic is generally not used.  ep_reset_busy_poll_napi_id()
is similarly always called, even when busy loop is not used.

Eliminate ep_reset_busy_poll_napi_id() and inline it inside
ep_busy_loop().  Make ep_busy_loop() return whether there are any events
available after the busy loop.  This will eliminate unnecessary loads and
branches, and simplifies the loop.

Link: https://lkml.kernel.org/r/20201106231635.3528496-6-soheil.kdev@gmail.com


Signed-off-by: default avatarSoheil Hassas Yeganeh <soheil@google.com>
Reviewed-by: default avatarEric Dumazet <edumazet@google.com>
Reviewed-by: default avatarWillem de Bruijn <willemb@google.com>
Reviewed-by: default avatarKhazhismel Kumykov <khazhy@google.com>
Cc: Guantao Liu <guantaol@google.com>
Cc: Linus Torvalds <torvalds@linux-foundation.org>
Signed-off-by: default avatarAndrew Morton <akpm@linux-foundation.org>
Signed-off-by: default avatarLinus Torvalds <torvalds@linux-foundation.org>
parent e411596d

fs/eventpoll.c

+17 −23
Original line number Diff line number Diff line
@@ -389,19 +389,24 @@ static bool ep_busy_loop_end(void *p, unsigned long start_time) 
 *

 * we must do our busy polling with irqs enabled

 */

static void ep_busy_loop(struct eventpoll *ep, int nonblock)

static bool ep_busy_loop(struct eventpoll *ep, int nonblock)

{

	unsigned int napi_id = READ_ONCE(ep->napi_id);



	if ((napi_id >= MIN_NAPI_ID) && net_busy_loop_on())

	if ((napi_id >= MIN_NAPI_ID) && net_busy_loop_on()) {

		napi_busy_loop(napi_id, nonblock ? NULL : ep_busy_loop_end, ep, false,

			       BUSY_POLL_BUDGET);

}



static inline void ep_reset_busy_poll_napi_id(struct eventpoll *ep)

{

	if (ep->napi_id)

		if (ep_events_available(ep))

			return true;

		/*

		 * Busy poll timed out.  Drop NAPI ID for now, we can add

		 * it back in when we have moved a socket with a valid NAPI

		 * ID onto the ready list.

		 */

		ep->napi_id = 0;

		return false;

	}

	return false;

}



/*
@@ -441,12 +446,9 @@ static inline void ep_set_busy_poll_napi_id(struct epitem *epi) 


#else



static inline void ep_busy_loop(struct eventpoll *ep, int nonblock)

{

}



static inline void ep_reset_busy_poll_napi_id(struct eventpoll *ep)

static inline bool ep_busy_loop(struct eventpoll *ep, int nonblock)

{

	return false;

}



static inline void ep_set_busy_poll_napi_id(struct epitem *epi)
@@ -1772,21 +1774,13 @@ static int ep_poll(struct eventpoll *ep, struct epoll_event __user *events, 
	}



fetch_events:



	if (!ep_events_available(ep))

		ep_busy_loop(ep, timed_out);



	eavail = ep_events_available(ep);

	if (!eavail)

		eavail = ep_busy_loop(ep, timed_out);



	if (eavail)

		goto send_events;



	/*

	 * Busy poll timed out.  Drop NAPI ID for now, we can add

	 * it back in when we have moved a socket with a valid NAPI

	 * ID onto the ready list.

	 */

	ep_reset_busy_poll_napi_id(ep);



	do {

		if (signal_pending(current))

			return -EINTR;

CRA Git | Maintained and supported by SUSTech CRA and CCSE