io-wq: briefly spin for new work after finishing work (e995d512) · Commits · 戴 / test

fs/io-wq.c

+22 −2

Original line number	Diff line number	Diff line
		@@ -491,26 +491,46 @@ next:
		} while (1);
		}

		static inline void io_worker_spin_for_work(struct io_wqe *wqe)
		{
		int i = 0;

		while (++i < 1000) {
		if (io_wqe_run_queue(wqe))
		break;
		if (need_resched())
		break;
		cpu_relax();
		}
		}

		static int io_wqe_worker(void *data)
		{
		struct io_worker *worker = data;
		struct io_wqe *wqe = worker->wqe;
		struct io_wq *wq = wqe->wq;
		bool did_work;

		io_worker_start(wqe, worker);

		did_work = false;
		while (!test_bit(IO_WQ_BIT_EXIT, &wq->state)) {
		set_current_state(TASK_INTERRUPTIBLE);
		loop:
		if (did_work)
		io_worker_spin_for_work(wqe);
		spin_lock_irq(&wqe->lock);
		if (io_wqe_run_queue(wqe)) {
		__set_current_state(TASK_RUNNING);
		io_worker_handle_work(worker);
		continue;
		did_work = true;
		goto loop;
		}
		did_work = false;
		/* drops the lock on success, retry */
		if (__io_worker_idle(wqe, worker)) {
		__release(&wqe->lock);
		continue;
		goto loop;
		}
		spin_unlock_irq(&wqe->lock);
		if (signal_pending(current))

+4 −3

Original line number	Diff line number	Diff line
		@@ -35,7 +35,8 @@ static inline void wq_list_add_tail(struct io_wq_work_node *node,
		struct io_wq_work_list *list)
		{
		if (!list->first) {
		list->first = list->last = node;
		list->last = node;
		WRITE_ONCE(list->first, node);
		} else {
		list->last->next = node;
		list->last = node;
		@@ -47,7 +48,7 @@ static inline void wq_node_del(struct io_wq_work_list *list,
		struct io_wq_work_node *prev)
		{
		if (node == list->first)
		list->first = node->next;
		WRITE_ONCE(list->first, node->next);
		if (node == list->last)
		list->last = prev;
		if (prev)
		@@ -58,7 +59,7 @@ static inline void wq_node_del(struct io_wq_work_list *list,
		#define wq_list_for_each(pos, prv, head) \
		for (pos = (head)->first, prv = NULL; pos; prv = pos, pos = (pos)->next)

		#define wq_list_empty(list) ((list)->first == NULL)
		#define wq_list_empty(list) (READ_ONCE((list)->first) == NULL)
		#define INIT_WQ_LIST(list) do { \
		(list)->first = NULL; \
		(list)->last = NULL; \