Commit 1f68f31b authored by Linus Torvalds's avatar Linus Torvalds
Browse files

Merge tag 'io_uring-5.8-2020-07-24' of git://git.kernel.dk/linux-block into master 

Pull io_uring fixes from Jens Axboe:

 - Fix discrepancy in how sqe->flags are treated for a few requests,
   this makes it consistent (Daniele)

 - Ensure that poll driven retry works with double waitqueue poll users

 - Fix a missing io_req_init_async() (Pavel)

* tag 'io_uring-5.8-2020-07-24' of git://git.kernel.dk/linux-block:
  io_uring: missed req_init_async() for IOSQE_ASYNC
  io_uring: always allow drain/link/hardlink/async sqe flags
  io_uring: ensure double poll additions work with both request types
parents 5a0b8af0 3e863ea3

fs/io_uring.c

+36 −25
Original line number Diff line number Diff line
@@ -605,6 +605,7 @@ enum { 


struct async_poll {

	struct io_poll_iocb	poll;

	struct io_poll_iocb	*double_poll;

	struct io_wq_work	work;

};
@@ -4159,9 +4160,9 @@ static bool io_poll_rewait(struct io_kiocb *req, struct io_poll_iocb *poll) 
	return false;

}



static void io_poll_remove_double(struct io_kiocb *req)

static void io_poll_remove_double(struct io_kiocb *req, void *data)

{

	struct io_poll_iocb *poll = (struct io_poll_iocb *) req->io;

	struct io_poll_iocb *poll = data;



	lockdep_assert_held(&req->ctx->completion_lock);
@@ -4181,7 +4182,7 @@ static void io_poll_complete(struct io_kiocb *req, __poll_t mask, int error) 
{

	struct io_ring_ctx *ctx = req->ctx;



	io_poll_remove_double(req);

	io_poll_remove_double(req, req->io);

	req->poll.done = true;

	io_cqring_fill_event(req, error ? error : mangle_poll(mask));

	io_commit_cqring(ctx);
@@ -4224,21 +4225,21 @@ static int io_poll_double_wake(struct wait_queue_entry *wait, unsigned mode, 
			       int sync, void *key)

{

	struct io_kiocb *req = wait->private;

	struct io_poll_iocb *poll = (struct io_poll_iocb *) req->io;

	struct io_poll_iocb *poll = req->apoll->double_poll;

	__poll_t mask = key_to_poll(key);



	/* for instances that support it check for an event match first: */

	if (mask && !(mask & poll->events))

		return 0;



	if (req->poll.head) {

	if (poll && poll->head) {

		bool done;



		spin_lock(&req->poll.head->lock);

		done = list_empty(&req->poll.wait.entry);

		spin_lock(&poll->head->lock);

		done = list_empty(&poll->wait.entry);

		if (!done)

			list_del_init(&req->poll.wait.entry);

		spin_unlock(&req->poll.head->lock);

			list_del_init(&poll->wait.entry);

		spin_unlock(&poll->head->lock);

		if (!done)

			__io_async_wake(req, poll, mask, io_poll_task_func);

	}
@@ -4258,7 +4259,8 @@ static void io_init_poll_iocb(struct io_poll_iocb *poll, __poll_t events, 
}



static void __io_queue_proc(struct io_poll_iocb *poll, struct io_poll_table *pt,

			    struct wait_queue_head *head)

			    struct wait_queue_head *head,

			    struct io_poll_iocb **poll_ptr)

{

	struct io_kiocb *req = pt->req;
@@ -4269,7 +4271,7 @@ static void __io_queue_proc(struct io_poll_iocb *poll, struct io_poll_table *pt, 
	 */

	if (unlikely(poll->head)) {

		/* already have a 2nd entry, fail a third attempt */

		if (req->io) {

		if (*poll_ptr) {

			pt->error = -EINVAL;

			return;

		}
@@ -4281,7 +4283,7 @@ static void __io_queue_proc(struct io_poll_iocb *poll, struct io_poll_table *pt, 
		io_init_poll_iocb(poll, req->poll.events, io_poll_double_wake);

		refcount_inc(&req->refs);

		poll->wait.private = req;

		req->io = (void *) poll;

		*poll_ptr = poll;

	}



	pt->error = 0;
@@ -4293,8 +4295,9 @@ static void io_async_queue_proc(struct file *file, struct wait_queue_head *head, 
			       struct poll_table_struct *p)

{

	struct io_poll_table *pt = container_of(p, struct io_poll_table, pt);

	struct async_poll *apoll = pt->req->apoll;



	__io_queue_proc(&pt->req->apoll->poll, pt, head);

	__io_queue_proc(&apoll->poll, pt, head, &apoll->double_poll);

}



static void io_sq_thread_drop_mm(struct io_ring_ctx *ctx)
@@ -4344,11 +4347,13 @@ static void io_async_task_func(struct callback_head *cb) 
		}

	}



	io_poll_remove_double(req, apoll->double_poll);

	spin_unlock_irq(&ctx->completion_lock);



	/* restore ->work in case we need to retry again */

	if (req->flags & REQ_F_WORK_INITIALIZED)

		memcpy(&req->work, &apoll->work, sizeof(req->work));

	kfree(apoll->double_poll);

	kfree(apoll);



	if (!canceled) {
@@ -4436,7 +4441,6 @@ static bool io_arm_poll_handler(struct io_kiocb *req) 
	struct async_poll *apoll;

	struct io_poll_table ipt;

	__poll_t mask, ret;

	bool had_io;



	if (!req->file || !file_can_poll(req->file))

		return false;
@@ -4448,11 +4452,11 @@ static bool io_arm_poll_handler(struct io_kiocb *req) 
	apoll = kmalloc(sizeof(*apoll), GFP_ATOMIC);

	if (unlikely(!apoll))

		return false;

	apoll->double_poll = NULL;



	req->flags |= REQ_F_POLLED;

	if (req->flags & REQ_F_WORK_INITIALIZED)

		memcpy(&apoll->work, &req->work, sizeof(req->work));

	had_io = req->io != NULL;



	io_get_req_task(req);

	req->apoll = apoll;
@@ -4470,13 +4474,11 @@ static bool io_arm_poll_handler(struct io_kiocb *req) 
	ret = __io_arm_poll_handler(req, &apoll->poll, &ipt, mask,

					io_async_wake);

	if (ret) {

		ipt.error = 0;

		/* only remove double add if we did it here */

		if (!had_io)

			io_poll_remove_double(req);

		io_poll_remove_double(req, apoll->double_poll);

		spin_unlock_irq(&ctx->completion_lock);

		if (req->flags & REQ_F_WORK_INITIALIZED)

			memcpy(&req->work, &apoll->work, sizeof(req->work));

		kfree(apoll->double_poll);

		kfree(apoll);

		return false;

	}
@@ -4507,11 +4509,13 @@ static bool io_poll_remove_one(struct io_kiocb *req) 
	bool do_complete;



	if (req->opcode == IORING_OP_POLL_ADD) {

		io_poll_remove_double(req);

		io_poll_remove_double(req, req->io);

		do_complete = __io_poll_remove_one(req, &req->poll);

	} else {

		struct async_poll *apoll = req->apoll;



		io_poll_remove_double(req, apoll->double_poll);



		/* non-poll requests have submit ref still */

		do_complete = __io_poll_remove_one(req, &apoll->poll);

		if (do_complete) {
@@ -4524,6 +4528,7 @@ static bool io_poll_remove_one(struct io_kiocb *req) 
			if (req->flags & REQ_F_WORK_INITIALIZED)

				memcpy(&req->work, &apoll->work,

				       sizeof(req->work));

			kfree(apoll->double_poll);

			kfree(apoll);

		}

	}
@@ -4624,7 +4629,7 @@ static void io_poll_queue_proc(struct file *file, struct wait_queue_head *head, 
{

	struct io_poll_table *pt = container_of(p, struct io_poll_table, pt);



	__io_queue_proc(&pt->req->poll, pt, head);

	__io_queue_proc(&pt->req->poll, pt, head, (struct io_poll_iocb **) &pt->req->io);

}



static int io_poll_add_prep(struct io_kiocb *req, const struct io_uring_sqe *sqe)
@@ -4732,7 +4737,9 @@ static int io_timeout_remove_prep(struct io_kiocb *req, 
{

	if (unlikely(req->ctx->flags & IORING_SETUP_IOPOLL))

		return -EINVAL;

	if (sqe->flags || sqe->ioprio || sqe->buf_index || sqe->len)

	if (unlikely(req->flags & (REQ_F_FIXED_FILE | REQ_F_BUFFER_SELECT)))

		return -EINVAL;

	if (sqe->ioprio || sqe->buf_index || sqe->len)

		return -EINVAL;



	req->timeout.addr = READ_ONCE(sqe->addr);
@@ -4910,8 +4917,9 @@ static int io_async_cancel_prep(struct io_kiocb *req, 
{

	if (unlikely(req->ctx->flags & IORING_SETUP_IOPOLL))

		return -EINVAL;

	if (sqe->flags || sqe->ioprio || sqe->off || sqe->len ||

	    sqe->cancel_flags)

	if (unlikely(req->flags & (REQ_F_FIXED_FILE | REQ_F_BUFFER_SELECT)))

		return -EINVAL;

	if (sqe->ioprio || sqe->off || sqe->len || sqe->cancel_flags)

		return -EINVAL;



	req->cancel.addr = READ_ONCE(sqe->addr);
@@ -4929,7 +4937,9 @@ static int io_async_cancel(struct io_kiocb *req) 
static int io_files_update_prep(struct io_kiocb *req,

				const struct io_uring_sqe *sqe)

{

	if (sqe->flags || sqe->ioprio || sqe->rw_flags)

	if (unlikely(req->flags & (REQ_F_FIXED_FILE | REQ_F_BUFFER_SELECT)))

		return -EINVAL;

	if (sqe->ioprio || sqe->rw_flags)

		return -EINVAL;



	req->files_update.offset = READ_ONCE(sqe->off);
@@ -5720,6 +5730,7 @@ fail_req: 
		 * Never try inline submit of IOSQE_ASYNC is set, go straight

		 * to async execution.

		 */

		io_req_init_async(req);

		req->work.flags |= IO_WQ_WORK_CONCURRENT;

		io_queue_async_work(req);

	} else {

CRA Git | Maintained and supported by SUSTech CRA and CCSE