Commit c8d1ba58 authored by Jens Axboe's avatar Jens Axboe
Browse files

io_uring: split work handling part of SQPOLL into helper 



This is done in preparation for handling more than one ctx, but it also
cleans up the code a bit since io_sq_thread() was a bit too unwieldy to
get a get overview on.

__io_sq_thread() is now the main handler, and it returns an enum sq_ret
that tells io_sq_thread() what it ended up doing. The parent then makes
a decision on idle, spinning, or work handling based on that.

Signed-off-by: default avatarJens Axboe <axboe@kernel.dk>
parent 3f0e64d0

fs/io_uring.c

+93 −84
Original line number Diff line number Diff line
@@ -6642,32 +6642,26 @@ static int io_sq_wake_function(struct wait_queue_entry *wqe, unsigned mode, 
	return ret;

}



static int io_sq_thread(void *data)

{

	struct io_ring_ctx *ctx = data;

	const struct cred *old_cred;

	unsigned long timeout;

	int ret = 0;



	init_wait(&ctx->sqo_wait_entry);

	ctx->sqo_wait_entry.func = io_sq_wake_function;



	complete(&ctx->sq_thread_comp);



	old_cred = override_creds(ctx->creds);

enum sq_ret {

	SQT_IDLE	= 1,

	SQT_SPIN	= 2,

	SQT_DID_WORK	= 4,

};



	timeout = jiffies + ctx->sq_thread_idle;

	while (!kthread_should_park()) {

static enum sq_ret __io_sq_thread(struct io_ring_ctx *ctx,

				  unsigned long start_jiffies)

{

	unsigned long timeout = start_jiffies + ctx->sq_thread_idle;

	unsigned int to_submit;

	int ret = 0;



again:

	if (!list_empty(&ctx->iopoll_list)) {

		unsigned nr_events = 0;



		mutex_lock(&ctx->uring_lock);

		if (!list_empty(&ctx->iopoll_list) && !need_resched())

			io_do_iopoll(ctx, &nr_events, 0);

			else

				timeout = jiffies + ctx->sq_thread_idle;

		mutex_unlock(&ctx->uring_lock);

	}
@@ -6695,11 +6689,8 @@ static int io_sq_thread(void *data) 
		 */

		if (!list_empty(&ctx->iopoll_list) || need_resched() ||

		    (!time_after(jiffies, timeout) && ret != -EBUSY &&

			    !percpu_ref_is_dying(&ctx->refs))) {

				io_run_task_work();

				cond_resched();

				continue;

			}

		    !percpu_ref_is_dying(&ctx->refs)))

			return SQT_SPIN;



		prepare_to_wait(ctx->sqo_wait, &ctx->sqo_wait_entry,

					TASK_INTERRUPTIBLE);
@@ -6714,38 +6705,56 @@ static int io_sq_thread(void *data) 
		if ((ctx->flags & IORING_SETUP_IOPOLL) &&

		    !list_empty_careful(&ctx->iopoll_list)) {

			finish_wait(ctx->sqo_wait, &ctx->sqo_wait_entry);

				continue;

			goto again;

		}



		io_ring_set_wakeup_flag(ctx);



		to_submit = io_sqring_entries(ctx);

			if (!to_submit || ret == -EBUSY) {

				if (kthread_should_park()) {

					finish_wait(ctx->sqo_wait, &ctx->sqo_wait_entry);

					break;

		if (!to_submit || ret == -EBUSY)

			return SQT_IDLE;

	}

				if (io_run_task_work()) {

					finish_wait(ctx->sqo_wait, &ctx->sqo_wait_entry);

					io_ring_clear_wakeup_flag(ctx);

					continue;

				}

				schedule();

				finish_wait(ctx->sqo_wait, &ctx->sqo_wait_entry);



				ret = 0;

				continue;

			}

	finish_wait(ctx->sqo_wait, &ctx->sqo_wait_entry);



	io_ring_clear_wakeup_flag(ctx);

		}



	mutex_lock(&ctx->uring_lock);

	if (likely(!percpu_ref_is_dying(&ctx->refs)))

		ret = io_submit_sqes(ctx, to_submit);

	mutex_unlock(&ctx->uring_lock);

		timeout = jiffies + ctx->sq_thread_idle;

	return SQT_DID_WORK;

}



static int io_sq_thread(void *data)

{

	struct io_ring_ctx *ctx = data;

	const struct cred *old_cred;

	unsigned long start_jiffies;



	init_wait(&ctx->sqo_wait_entry);

	ctx->sqo_wait_entry.func = io_sq_wake_function;



	complete(&ctx->sq_thread_comp);



	old_cred = override_creds(ctx->creds);



	start_jiffies = jiffies;

	while (!kthread_should_park()) {

		enum sq_ret ret;



		ret = __io_sq_thread(ctx, start_jiffies);

		switch (ret) {

		case SQT_IDLE:

			schedule();

			start_jiffies = jiffies;

			continue;

		case SQT_SPIN:

			io_run_task_work();

			cond_resched();

			fallthrough;

		case SQT_DID_WORK:

			continue;

		}

	}



	io_run_task_work();

CRA Git | Maintained and supported by SUSTech CRA and CCSE