Commit dc026a73 authored by Pavel Begunkov's avatar Pavel Begunkov Committed by Jens Axboe
Browse files

io-wq: shuffle io_worker_handle_work() code 



This is a preparation patch, it adds some helpers and makes
the next patches cleaner.

- extract io_impersonate_work() and io_assign_current_work()
- replace @next label with nested do-while
- move put_work() right after NULL'ing cur_work.

Signed-off-by: default avatarPavel Begunkov <asml.silence@gmail.com>
Signed-off-by: default avatarJens Axboe <axboe@kernel.dk>
parent 7a743e22

fs/io-wq.c

+64 −59
Original line number Diff line number Diff line
@@ -440,14 +440,43 @@ static void io_wq_switch_creds(struct io_worker *worker, 
		worker->saved_creds = old_creds;

}



static void io_impersonate_work(struct io_worker *worker,

				struct io_wq_work *work)

{

	if (work->files && current->files != work->files) {

		task_lock(current);

		current->files = work->files;

		task_unlock(current);

	}

	if (work->fs && current->fs != work->fs)

		current->fs = work->fs;

	if (work->mm != worker->mm)

		io_wq_switch_mm(worker, work);

	if (worker->cur_creds != work->creds)

		io_wq_switch_creds(worker, work);

}



static void io_assign_current_work(struct io_worker *worker,

				   struct io_wq_work *work)

{

	/* flush pending signals before assigning new work */

	if (signal_pending(current))

		flush_signals(current);

	cond_resched();



	spin_lock_irq(&worker->lock);

	worker->cur_work = work;

	spin_unlock_irq(&worker->lock);

}



static void io_worker_handle_work(struct io_worker *worker)

	__releases(wqe->lock)

{

	struct io_wq_work *work, *old_work = NULL, *put_work = NULL;

	struct io_wqe *wqe = worker->wqe;

	struct io_wq *wq = wqe->wq;



	do {

		struct io_wq_work *work, *old_work;

		unsigned hash = -1U;



		/*
@@ -464,43 +493,23 @@ static void io_worker_handle_work(struct io_worker *worker) 
			wqe->flags |= IO_WQE_FLAG_STALLED;



		spin_unlock_irq(&wqe->lock);

		if (put_work && wq->put_work)

			wq->put_work(old_work);

		if (!work)

			break;

next:

		/* flush any pending signals before assigning new work */

		if (signal_pending(current))

			flush_signals(current);



		cond_resched();



		spin_lock_irq(&worker->lock);

		worker->cur_work = work;

		spin_unlock_irq(&worker->lock);

		/* handle a whole dependent link */

		do {

			io_assign_current_work(worker, work);

			io_impersonate_work(worker, work);



		if (work->files && current->files != work->files) {

			task_lock(current);

			current->files = work->files;

			task_unlock(current);

		}

		if (work->fs && current->fs != work->fs)

			current->fs = work->fs;

		if (work->mm != worker->mm)

			io_wq_switch_mm(worker, work);

		if (worker->cur_creds != work->creds)

			io_wq_switch_creds(worker, work);

			/*

		 * OK to set IO_WQ_WORK_CANCEL even for uncancellable work,

		 * the worker function will do the right thing.

			 * OK to set IO_WQ_WORK_CANCEL even for uncancellable

			 * work, the worker function will do the right thing.

			 */

			if (test_bit(IO_WQ_BIT_CANCEL, &wq->state))

				work->flags |= IO_WQ_WORK_CANCEL;



		if (wq->get_work) {

			put_work = work;

			if (wq->get_work)

				wq->get_work(work);

		}



			old_work = work;

			work->func(&work);
@@ -509,24 +518,20 @@ next: 
			worker->cur_work = NULL;

			spin_unlock_irq(&worker->lock);



		spin_lock_irq(&wqe->lock);

			if (wq->put_work)

				wq->put_work(old_work);



			if (hash != -1U) {

			wqe->hash_map &= ~BIT(hash);

				spin_lock_irq(&wqe->lock);

				wqe->hash_map &= ~BIT_ULL(hash);

				wqe->flags &= ~IO_WQE_FLAG_STALLED;

		}

		if (work && work != old_work) {

				spin_unlock_irq(&wqe->lock);



			if (put_work && wq->put_work) {

				wq->put_work(put_work);

				put_work = NULL;

			}



			/* dependent work not hashed */

				/* dependent work is not hashed */

				hash = -1U;

			goto next;

			}

		} while (work && work != old_work);



		spin_lock_irq(&wqe->lock);

	} while (1);

}

CRA Git | Maintained and supported by SUSTech CRA and CCSE