Commit 4dd2824d authored by Pavel Begunkov's avatar Pavel Begunkov Committed by Jens Axboe
Browse files

io_uring: lazy get task 



There will be multiple places where req->task is used, so refcount-pin
it lazily with introduced *io_{get,put}_req_task(). We need to always
have valid ->task for cancellation reasons, but don't care about pinning
it in some cases. That's why it sets req->task in io_req_init() and
implements get/put laziness with a flag.

This also removes using @current from polling io_arm_poll_handler(),
etc., but doesn't change observable behaviour.

Signed-off-by: default avatarPavel Begunkov <asml.silence@gmail.com>
Signed-off-by: default avatarJens Axboe <axboe@kernel.dk>
parent 67c4d9e6

fs/io_uring.c

+22 −8
Original line number Diff line number Diff line
@@ -541,6 +541,7 @@ enum { 
	REQ_F_NO_FILE_TABLE_BIT,

	REQ_F_QUEUE_TIMEOUT_BIT,

	REQ_F_WORK_INITIALIZED_BIT,

	REQ_F_TASK_PINNED_BIT,



	/* not a real bit, just to check we're not overflowing the space */

	__REQ_F_LAST_BIT,
@@ -598,6 +599,8 @@ enum { 
	REQ_F_QUEUE_TIMEOUT	= BIT(REQ_F_QUEUE_TIMEOUT_BIT),

	/* io_wq_work is initialized */

	REQ_F_WORK_INITIALIZED	= BIT(REQ_F_WORK_INITIALIZED_BIT),

	/* req->task is refcounted */

	REQ_F_TASK_PINNED	= BIT(REQ_F_TASK_PINNED_BIT),

};



struct async_poll {
@@ -910,6 +913,21 @@ struct sock *io_uring_get_socket(struct file *file) 
}

EXPORT_SYMBOL(io_uring_get_socket);



static void io_get_req_task(struct io_kiocb *req)

{

	if (req->flags & REQ_F_TASK_PINNED)

		return;

	get_task_struct(req->task);

	req->flags |= REQ_F_TASK_PINNED;

}



/* not idempotent -- it doesn't clear REQ_F_TASK_PINNED */

static void __io_put_req_task(struct io_kiocb *req)

{

	if (req->flags & REQ_F_TASK_PINNED)

		put_task_struct(req->task);

}



static void io_file_put_work(struct work_struct *work);



/*
@@ -1399,9 +1417,7 @@ static void __io_req_aux_free(struct io_kiocb *req) 
	kfree(req->io);

	if (req->file)

		io_put_file(req, req->file, (req->flags & REQ_F_FIXED_FILE));

	if (req->task)

		put_task_struct(req->task);



	__io_put_req_task(req);

	io_req_work_drop_env(req);

}
@@ -4367,8 +4383,7 @@ static bool io_arm_poll_handler(struct io_kiocb *req) 
		memcpy(&apoll->work, &req->work, sizeof(req->work));

	had_io = req->io != NULL;



	get_task_struct(current);

	req->task = current;

	io_get_req_task(req);

	req->apoll = apoll;

	INIT_HLIST_NODE(&req->hash_node);
@@ -4556,8 +4571,7 @@ static int io_poll_add_prep(struct io_kiocb *req, const struct io_uring_sqe *sqe 
	events = READ_ONCE(sqe->poll_events);

	poll->events = demangle_poll(events) | EPOLLERR | EPOLLHUP;



	get_task_struct(current);

	req->task = current;

	io_get_req_task(req);

	return 0;

}
@@ -5818,7 +5832,7 @@ static int io_init_req(struct io_ring_ctx *ctx, struct io_kiocb *req, 
	req->flags = 0;

	/* one is dropped after submission, the other at completion */

	refcount_set(&req->refs, 2);

	req->task = NULL;

	req->task = current;

	req->result = 0;



	if (unlikely(req->opcode >= IORING_OP_LAST))

CRA Git | Maintained and supported by SUSTech CRA and CCSE