Commit 23b3628e authored by Xiaoguang Wang's avatar Xiaoguang Wang Committed by Jens Axboe
Browse files

io_uring: clear IORING_SQ_NEED_WAKEUP after executing task works 



In io_sq_thread(), if there are task works to handle, current codes
will skip schedule() and go on polling sq again, but forget to clear
IORING_SQ_NEED_WAKEUP flag, fix this issue. Also add two helpers to
set and clear IORING_SQ_NEED_WAKEUP flag,

Signed-off-by: default avatarXiaoguang Wang <xiaoguang.wang@linux.alibaba.com>
Signed-off-by: default avatarJens Axboe <axboe@kernel.dk>
parent 5af1d13e

fs/io_uring.c

+19 −10
Original line number Diff line number Diff line
@@ -6344,6 +6344,21 @@ fail_req: 
	return submitted;

}



static inline void io_ring_set_wakeup_flag(struct io_ring_ctx *ctx)

{

	/* Tell userspace we may need a wakeup call */

	spin_lock_irq(&ctx->completion_lock);

	ctx->rings->sq_flags |= IORING_SQ_NEED_WAKEUP;

	spin_unlock_irq(&ctx->completion_lock);

}



static inline void io_ring_clear_wakeup_flag(struct io_ring_ctx *ctx)

{

	spin_lock_irq(&ctx->completion_lock);

	ctx->rings->sq_flags &= ~IORING_SQ_NEED_WAKEUP;

	spin_unlock_irq(&ctx->completion_lock);

}



static int io_sq_thread(void *data)

{

	struct io_ring_ctx *ctx = data;
@@ -6417,10 +6432,7 @@ static int io_sq_thread(void *data) 
				continue;

			}



			/* Tell userspace we may need a wakeup call */

			spin_lock_irq(&ctx->completion_lock);

			ctx->rings->sq_flags |= IORING_SQ_NEED_WAKEUP;

			spin_unlock_irq(&ctx->completion_lock);

			io_ring_set_wakeup_flag(ctx);



			to_submit = io_sqring_entries(ctx);

			if (!to_submit || ret == -EBUSY) {
@@ -6430,6 +6442,7 @@ static int io_sq_thread(void *data) 
				}

				if (io_run_task_work()) {

					finish_wait(&ctx->sqo_wait, &wait);

					io_ring_clear_wakeup_flag(ctx);

					continue;

				}

				if (signal_pending(current))
@@ -6437,17 +6450,13 @@ static int io_sq_thread(void *data) 
				schedule();

				finish_wait(&ctx->sqo_wait, &wait);



				spin_lock_irq(&ctx->completion_lock);

				ctx->rings->sq_flags &= ~IORING_SQ_NEED_WAKEUP;

				spin_unlock_irq(&ctx->completion_lock);

				io_ring_clear_wakeup_flag(ctx);

				ret = 0;

				continue;

			}

			finish_wait(&ctx->sqo_wait, &wait);



			spin_lock_irq(&ctx->completion_lock);

			ctx->rings->sq_flags &= ~IORING_SQ_NEED_WAKEUP;

			spin_unlock_irq(&ctx->completion_lock);

			io_ring_clear_wakeup_flag(ctx);

		}



		mutex_lock(&ctx->uring_lock);

CRA Git | Maintained and supported by SUSTech CRA and CCSE