Commit 2b2ed975 authored by Jens Axboe's avatar Jens Axboe
Browse files

io_uring: fix bad inflight accounting for SETUP_IOPOLL|SETUP_SQTHREAD 



We currently assume that submissions from the sqthread are successful,
and if IO polling is enabled, we use that value for knowing how many
completions to look for. But if we overflowed the CQ ring or some
requests simply got errored and already completed, they won't be
available for polling.

For the case of IO polling and SQTHREAD usage, look at the pending
poll list. If it ever hits empty then we know that we don't have
anymore pollable requests inflight. For that case, simply reset
the inflight count to zero.

Reported-by: default avatarPavel Begunkov <asml.silence@gmail.com>
Reviewed-by: default avatarPavel Begunkov <asml.silence@gmail.com>
Signed-off-by: default avatarJens Axboe <axboe@kernel.dk>
parent 498ccd9e

fs/io_uring.c

+32 −12
Original line number Diff line number Diff line
@@ -874,19 +874,11 @@ static void io_iopoll_reap_events(struct io_ring_ctx *ctx) 
	mutex_unlock(&ctx->uring_lock);

}



static int io_iopoll_check(struct io_ring_ctx *ctx, unsigned *nr_events,

static int __io_iopoll_check(struct io_ring_ctx *ctx, unsigned *nr_events,

			    long min)

{

	int iters, ret = 0;



	/*

	 * We disallow the app entering submit/complete with polling, but we

	 * still need to lock the ring to prevent racing with polled issue

	 * that got punted to a workqueue.

	 */

	mutex_lock(&ctx->uring_lock);

	int iters = 0, ret = 0;



	iters = 0;

	do {

		int tmin = 0;
@@ -922,6 +914,21 @@ static int io_iopoll_check(struct io_ring_ctx *ctx, unsigned *nr_events, 
		ret = 0;

	} while (min && !*nr_events && !need_resched());



	return ret;

}



static int io_iopoll_check(struct io_ring_ctx *ctx, unsigned *nr_events,

			   long min)

{

	int ret;



	/*

	 * We disallow the app entering submit/complete with polling, but we

	 * still need to lock the ring to prevent racing with polled issue

	 * that got punted to a workqueue.

	 */

	mutex_lock(&ctx->uring_lock);

	ret = __io_iopoll_check(ctx, nr_events, min);

	mutex_unlock(&ctx->uring_lock);

	return ret;

}
@@ -2657,7 +2664,20 @@ static int io_sq_thread(void *data) 
			unsigned nr_events = 0;



			if (ctx->flags & IORING_SETUP_IOPOLL) {

				io_iopoll_check(ctx, &nr_events, 0);

				/*

				 * inflight is the count of the maximum possible

				 * entries we submitted, but it can be smaller

				 * if we dropped some of them. If we don't have

				 * poll entries available, then we know that we

				 * have nothing left to poll for. Reset the

				 * inflight count to zero in that case.

				 */

				mutex_lock(&ctx->uring_lock);

				if (!list_empty(&ctx->poll_list))

					__io_iopoll_check(ctx, &nr_events, 0);

				else

					inflight = 0;

				mutex_unlock(&ctx->uring_lock);

			} else {

				/*

				 * Normal IO, just pretend everything completed.

CRA Git | Maintained and supported by SUSTech CRA and CCSE