io_uring: use io-wq manager as backup task if task is exiting (aa96bf8a) · Commits · 戴 / test

fs/io-wq.c

+12 −0

Original line number	Diff line number	Diff line
		@@ -17,6 +17,7 @@
		#include <linux/kthread.h>
		#include <linux/rculist_nulls.h>
		#include <linux/fs_struct.h>
		#include <linux/task_work.h>

		#include "io-wq.h"

		@@ -716,6 +717,9 @@ static int io_wq_manager(void *data)
		complete(&wq->done);

		while (!kthread_should_stop()) {
		if (current->task_works)
		task_work_run();

		for_each_node(node) {
		struct io_wqe *wqe = wq->wqes[node];
		bool fork_worker[2] = { false, false };
		@@ -738,6 +742,9 @@ static int io_wq_manager(void *data)
		schedule_timeout(HZ);
		}

		if (current->task_works)
		task_work_run();

		return 0;
		err:
		set_bit(IO_WQ_BIT_ERROR, &wq->state);
		@@ -1124,3 +1131,8 @@ void io_wq_destroy(struct io_wq *wq)
		if (refcount_dec_and_test(&wq->use_refs))
		__io_wq_destroy(wq);
		}

		struct task_struct io_wq_get_task(struct io_wq wq)
		{
		return wq->manager;
		}

+2 −0

Original line number	Diff line number	Diff line
		@@ -136,6 +136,8 @@ typedef bool (work_cancel_fn)(struct io_wq_work , void );
		enum io_wq_cancel io_wq_cancel_cb(struct io_wq wq, work_cancel_fn cancel,
		void *data);

		struct task_struct io_wq_get_task(struct io_wq wq);

		#if defined(CONFIG_IO_WQ)
		extern void io_wq_worker_sleeping(struct task_struct *);
		extern void io_wq_worker_running(struct task_struct *);

+9 −4

Original line number	Diff line number	Diff line
		@@ -4120,6 +4120,7 @@ static int __io_async_wake(struct io_kiocb req, struct io_poll_iocb poll,
		__poll_t mask, task_work_func_t func)
		{
		struct task_struct *tsk;
		int ret;

		/* for instances that support it check for an event match first: */
		if (mask && !(mask & poll->events))
		@@ -4133,11 +4134,15 @@ static int __io_async_wake(struct io_kiocb req, struct io_poll_iocb poll,
		req->result = mask;
		init_task_work(&req->task_work, func);
		/*
		* If this fails, then the task is exiting. If that is the case, then
		* the exit check will ultimately cancel these work items. Hence we
		* don't need to check here and handle it specifically.
		* If this fails, then the task is exiting. Punt to one of the io-wq
		* threads to ensure the work gets run, we can't always rely on exit
		* cancelation taking care of this.
		*/
		ret = task_work_add(tsk, &req->task_work, true);
		if (unlikely(ret)) {
		tsk = io_wq_get_task(req->ctx->io_wq);
		task_work_add(tsk, &req->task_work, true);
		}
		wake_up_process(tsk);
		return 1;
		}