Commit 18bceab1 authored by Jens Axboe's avatar Jens Axboe
Browse files

io_uring: allow POLL_ADD with double poll_wait() users 



Some file descriptors use separate waitqueues for their f_ops->poll()
handler, most commonly one for read and one for write. The io_uring
poll implementation doesn't work with that, as the 2nd poll_wait()
call will cause the io_uring poll request to -EINVAL.

This affects (at least) tty devices and /dev/random as well. This is a
big problem for event loops where some file descriptors work, and others
don't.

With this fix, io_uring handles multiple waitqueues.

Signed-off-by: default avatarJens Axboe <axboe@kernel.dk>
parent 4a38aed2

fs/io_uring.c

+146 −72
Original line number Diff line number Diff line
@@ -4099,27 +4099,6 @@ struct io_poll_table { 
	int error;

};



static void __io_queue_proc(struct io_poll_iocb *poll, struct io_poll_table *pt,

			    struct wait_queue_head *head)

{

	if (unlikely(poll->head)) {

		pt->error = -EINVAL;

		return;

	}



	pt->error = 0;

	poll->head = head;

	add_wait_queue(head, &poll->wait);

}



static void io_async_queue_proc(struct file *file, struct wait_queue_head *head,

			       struct poll_table_struct *p)

{

	struct io_poll_table *pt = container_of(p, struct io_poll_table, pt);



	__io_queue_proc(&pt->req->apoll->poll, pt, head);

}



static int __io_async_wake(struct io_kiocb *req, struct io_poll_iocb *poll,

			   __poll_t mask, task_work_func_t func)

{
@@ -4171,6 +4150,144 @@ static bool io_poll_rewait(struct io_kiocb *req, struct io_poll_iocb *poll) 
	return false;

}



static void io_poll_remove_double(struct io_kiocb *req)

{

	struct io_poll_iocb *poll = (struct io_poll_iocb *) req->io;



	lockdep_assert_held(&req->ctx->completion_lock);



	if (poll && poll->head) {

		struct wait_queue_head *head = poll->head;



		spin_lock(&head->lock);

		list_del_init(&poll->wait.entry);

		if (poll->wait.private)

			refcount_dec(&req->refs);

		poll->head = NULL;

		spin_unlock(&head->lock);

	}

}



static void io_poll_complete(struct io_kiocb *req, __poll_t mask, int error)

{

	struct io_ring_ctx *ctx = req->ctx;



	io_poll_remove_double(req);

	req->poll.done = true;

	io_cqring_fill_event(req, error ? error : mangle_poll(mask));

	io_commit_cqring(ctx);

}



static void io_poll_task_handler(struct io_kiocb *req, struct io_kiocb **nxt)

{

	struct io_ring_ctx *ctx = req->ctx;



	if (io_poll_rewait(req, &req->poll)) {

		spin_unlock_irq(&ctx->completion_lock);

		return;

	}



	hash_del(&req->hash_node);

	io_poll_complete(req, req->result, 0);

	req->flags |= REQ_F_COMP_LOCKED;

	io_put_req_find_next(req, nxt);

	spin_unlock_irq(&ctx->completion_lock);



	io_cqring_ev_posted(ctx);

}



static void io_poll_task_func(struct callback_head *cb)

{

	struct io_kiocb *req = container_of(cb, struct io_kiocb, task_work);

	struct io_kiocb *nxt = NULL;



	io_poll_task_handler(req, &nxt);

	if (nxt) {

		struct io_ring_ctx *ctx = nxt->ctx;



		mutex_lock(&ctx->uring_lock);

		__io_queue_sqe(nxt, NULL);

		mutex_unlock(&ctx->uring_lock);

	}

}



static int io_poll_double_wake(struct wait_queue_entry *wait, unsigned mode,

			       int sync, void *key)

{

	struct io_kiocb *req = wait->private;

	struct io_poll_iocb *poll = (struct io_poll_iocb *) req->io;

	__poll_t mask = key_to_poll(key);



	/* for instances that support it check for an event match first: */

	if (mask && !(mask & poll->events))

		return 0;



	if (req->poll.head) {

		bool done;



		spin_lock(&req->poll.head->lock);

		done = list_empty(&req->poll.wait.entry);

		if (!done)

			list_del_init(&req->poll.wait.entry);

		spin_unlock(&req->poll.head->lock);

		if (!done)

			__io_async_wake(req, poll, mask, io_poll_task_func);

	}

	refcount_dec(&req->refs);

	return 1;

}



static void io_init_poll_iocb(struct io_poll_iocb *poll, __poll_t events,

			      wait_queue_func_t wake_func)

{

	poll->head = NULL;

	poll->done = false;

	poll->canceled = false;

	poll->events = events;

	INIT_LIST_HEAD(&poll->wait.entry);

	init_waitqueue_func_entry(&poll->wait, wake_func);

}



static void __io_queue_proc(struct io_poll_iocb *poll, struct io_poll_table *pt,

			    struct wait_queue_head *head)

{

	struct io_kiocb *req = pt->req;



	/*

	 * If poll->head is already set, it's because the file being polled

	 * uses multiple waitqueues for poll handling (eg one for read, one

	 * for write). Setup a separate io_poll_iocb if this happens.

	 */

	if (unlikely(poll->head)) {

		/* already have a 2nd entry, fail a third attempt */

		if (req->io) {

			pt->error = -EINVAL;

			return;

		}

		poll = kmalloc(sizeof(*poll), GFP_ATOMIC);

		if (!poll) {

			pt->error = -ENOMEM;

			return;

		}

		io_init_poll_iocb(poll, req->poll.events, io_poll_double_wake);

		refcount_inc(&req->refs);

		poll->wait.private = req;

		req->io = (void *) poll;

	}



	pt->error = 0;

	poll->head = head;

	add_wait_queue(head, &poll->wait);

}



static void io_async_queue_proc(struct file *file, struct wait_queue_head *head,

			       struct poll_table_struct *p)

{

	struct io_poll_table *pt = container_of(p, struct io_poll_table, pt);



	__io_queue_proc(&pt->req->apoll->poll, pt, head);

}



static void io_async_task_func(struct callback_head *cb)

{

	struct io_kiocb *req = container_of(cb, struct io_kiocb, task_work);
@@ -4246,18 +4363,13 @@ static __poll_t __io_arm_poll_handler(struct io_kiocb *req, 
	bool cancel = false;



	poll->file = req->file;

	poll->head = NULL;

	poll->done = poll->canceled = false;

	poll->events = mask;

	io_init_poll_iocb(poll, mask, wake_func);

	poll->wait.private = req;



	ipt->pt._key = mask;

	ipt->req = req;

	ipt->error = -EINVAL;



	INIT_LIST_HEAD(&poll->wait.entry);

	init_waitqueue_func_entry(&poll->wait, wake_func);

	poll->wait.private = req;



	mask = vfs_poll(req->file, &ipt->pt) & poll->events;



	spin_lock_irq(&ctx->completion_lock);
@@ -4288,6 +4400,7 @@ static bool io_arm_poll_handler(struct io_kiocb *req) 
	struct async_poll *apoll;

	struct io_poll_table ipt;

	__poll_t mask, ret;

	bool had_io;



	if (!req->file || !file_can_poll(req->file))

		return false;
@@ -4302,6 +4415,7 @@ static bool io_arm_poll_handler(struct io_kiocb *req) 


	req->flags |= REQ_F_POLLED;

	memcpy(&apoll->work, &req->work, sizeof(req->work));

	had_io = req->io != NULL;



	get_task_struct(current);

	req->task = current;
@@ -4321,7 +4435,9 @@ static bool io_arm_poll_handler(struct io_kiocb *req) 
					io_async_wake);

	if (ret) {

		ipt.error = 0;

		apoll->poll.done = true;

		/* only remove double add if we did it here */

		if (!had_io)

			io_poll_remove_double(req);

		spin_unlock_irq(&ctx->completion_lock);

		memcpy(&req->work, &apoll->work, sizeof(req->work));

		kfree(apoll);
@@ -4354,6 +4470,7 @@ static bool io_poll_remove_one(struct io_kiocb *req) 
	bool do_complete;



	if (req->opcode == IORING_OP_POLL_ADD) {

		io_poll_remove_double(req);

		do_complete = __io_poll_remove_one(req, &req->poll);

	} else {

		apoll = req->apoll;
@@ -4455,49 +4572,6 @@ static int io_poll_remove(struct io_kiocb *req) 
	return 0;

}



static void io_poll_complete(struct io_kiocb *req, __poll_t mask, int error)

{

	struct io_ring_ctx *ctx = req->ctx;



	req->poll.done = true;

	io_cqring_fill_event(req, error ? error : mangle_poll(mask));

	io_commit_cqring(ctx);

}



static void io_poll_task_handler(struct io_kiocb *req, struct io_kiocb **nxt)

{

	struct io_ring_ctx *ctx = req->ctx;

	struct io_poll_iocb *poll = &req->poll;



	if (io_poll_rewait(req, poll)) {

		spin_unlock_irq(&ctx->completion_lock);

		return;

	}



	hash_del(&req->hash_node);

	io_poll_complete(req, req->result, 0);

	req->flags |= REQ_F_COMP_LOCKED;

	io_put_req_find_next(req, nxt);

	spin_unlock_irq(&ctx->completion_lock);



	io_cqring_ev_posted(ctx);

}



static void io_poll_task_func(struct callback_head *cb)

{

	struct io_kiocb *req = container_of(cb, struct io_kiocb, task_work);

	struct io_kiocb *nxt = NULL;



	io_poll_task_handler(req, &nxt);

	if (nxt) {

		struct io_ring_ctx *ctx = nxt->ctx;



		mutex_lock(&ctx->uring_lock);

		__io_queue_sqe(nxt, NULL);

		mutex_unlock(&ctx->uring_lock);

	}

}



static int io_poll_wake(struct wait_queue_entry *wait, unsigned mode, int sync,

			void *key)

{

CRA Git | Maintained and supported by SUSTech CRA and CCSE