Commit 7e84e1c7 authored by Stefano Garzarella's avatar Stefano Garzarella Committed by Jens Axboe
Browse files

io_uring: allow disabling rings during the creation 



This patch adds a new IORING_SETUP_R_DISABLED flag to start the
rings disabled, allowing the user to register restrictions,
buffers, files, before to start processing SQEs.

When IORING_SETUP_R_DISABLED is set, SQE are not processed and
SQPOLL kthread is not started.

The restrictions registration are allowed only when the rings
are disable to prevent concurrency issue while processing SQEs.

The rings can be enabled using IORING_REGISTER_ENABLE_RINGS
opcode with io_uring_register(2).

Suggested-by: default avatarJens Axboe <axboe@kernel.dk>
Signed-off-by: default avatarStefano Garzarella <sgarzare@redhat.com>
Reviewed-by: default avatarKees Cook <keescook@chromium.org>
Signed-off-by: default avatarJens Axboe <axboe@kernel.dk>
parent 21b55dbc

fs/io_uring.c

+54 −7
Original line number Diff line number Diff line
@@ -227,6 +227,7 @@ struct io_restriction { 
	DECLARE_BITMAP(sqe_op, IORING_OP_LAST);

	u8 sqe_flags_allowed;

	u8 sqe_flags_required;

	bool registered;

};



struct io_ring_ctx {
@@ -6910,6 +6911,14 @@ static int io_sqe_files_unregister(struct io_ring_ctx *ctx) 
static void io_sq_thread_stop(struct io_ring_ctx *ctx)

{

	if (ctx->sqo_thread) {

		/*

		 * We may arrive here from the error branch in

		 * io_sq_offload_create() where the kthread is created

		 * without being waked up, thus wake it up now to make

		 * sure the wait will complete.

		 */

		wake_up_process(ctx->sqo_thread);



		wait_for_completion(&ctx->sq_thread_comp);

		/*

		 * The park is a bit of a work-around, without it we get
@@ -7581,7 +7590,7 @@ void __io_uring_free(struct task_struct *tsk) 
	tsk->io_uring = NULL;

}



static int io_sq_offload_start(struct io_ring_ctx *ctx,

static int io_sq_offload_create(struct io_ring_ctx *ctx,

				struct io_uring_params *p)

{

	int ret;
@@ -7619,7 +7628,6 @@ static int io_sq_offload_start(struct io_ring_ctx *ctx, 
		ret = io_uring_alloc_task_context(ctx->sqo_thread);

		if (ret)

			goto err;

		wake_up_process(ctx->sqo_thread);

	} else if (p->flags & IORING_SETUP_SQ_AFF) {

		/* Can't have SQ_AFF without SQPOLL */

		ret = -EINVAL;
@@ -7636,6 +7644,12 @@ err: 
	return ret;

}



static void io_sq_offload_start(struct io_ring_ctx *ctx)

{

	if ((ctx->flags & IORING_SETUP_SQPOLL) && ctx->sqo_thread)

		wake_up_process(ctx->sqo_thread);

}



static inline void __io_unaccount_mem(struct user_struct *user,

				      unsigned long nr_pages)

{
@@ -8633,6 +8647,10 @@ SYSCALL_DEFINE6(io_uring_enter, unsigned int, fd, u32, to_submit, 
	if (!percpu_ref_tryget(&ctx->refs))

		goto out_fput;



	ret = -EBADFD;

	if (ctx->flags & IORING_SETUP_R_DISABLED)

		goto out;



	/*

	 * For SQ polling, the thread will do all submissions and completions.

	 * Just return the requested submit count, and wake the thread if
@@ -8975,10 +8993,13 @@ static int io_uring_create(unsigned entries, struct io_uring_params *p, 
	if (ret)

		goto err;



	ret = io_sq_offload_start(ctx, p);

	ret = io_sq_offload_create(ctx, p);

	if (ret)

		goto err;



	if (!(p->flags & IORING_SETUP_R_DISABLED))

		io_sq_offload_start(ctx);



	memset(&p->sq_off, 0, sizeof(p->sq_off));

	p->sq_off.head = offsetof(struct io_rings, sq.head);

	p->sq_off.tail = offsetof(struct io_rings, sq.tail);
@@ -9041,7 +9062,8 @@ static long io_uring_setup(u32 entries, struct io_uring_params __user *params) 


	if (p.flags & ~(IORING_SETUP_IOPOLL | IORING_SETUP_SQPOLL |

			IORING_SETUP_SQ_AFF | IORING_SETUP_CQSIZE |

			IORING_SETUP_CLAMP | IORING_SETUP_ATTACH_WQ))

			IORING_SETUP_CLAMP | IORING_SETUP_ATTACH_WQ |

			IORING_SETUP_R_DISABLED))

		return -EINVAL;



	return  io_uring_create(entries, &p, params);
@@ -9124,8 +9146,12 @@ static int io_register_restrictions(struct io_ring_ctx *ctx, void __user *arg, 
	size_t size;

	int i, ret;



	/* Restrictions allowed only if rings started disabled */

	if (!(ctx->flags & IORING_SETUP_R_DISABLED))

		return -EBADFD;



	/* We allow only a single restrictions registration */

	if (ctx->restricted)

	if (ctx->restrictions.registered)

		return -EBUSY;



	if (!arg || nr_args > IORING_MAX_RESTRICTIONS)
@@ -9177,12 +9203,27 @@ out: 
	if (ret != 0)

		memset(&ctx->restrictions, 0, sizeof(ctx->restrictions));

	else

		ctx->restricted = 1;

		ctx->restrictions.registered = true;



	kfree(res);

	return ret;

}



static int io_register_enable_rings(struct io_ring_ctx *ctx)

{

	if (!(ctx->flags & IORING_SETUP_R_DISABLED))

		return -EBADFD;



	if (ctx->restrictions.registered)

		ctx->restricted = 1;



	ctx->flags &= ~IORING_SETUP_R_DISABLED;



	io_sq_offload_start(ctx);



	return 0;

}



static bool io_register_op_must_quiesce(int op)

{

	switch (op) {
@@ -9304,6 +9345,12 @@ static int __io_uring_register(struct io_ring_ctx *ctx, unsigned opcode, 
			break;

		ret = io_unregister_personality(ctx, nr_args);

		break;

	case IORING_REGISTER_ENABLE_RINGS:

		ret = -EINVAL;

		if (arg || nr_args)

			break;

		ret = io_register_enable_rings(ctx);

		break;

	case IORING_REGISTER_RESTRICTIONS:

		ret = io_register_restrictions(ctx, arg, nr_args);

		break;

include/uapi/linux/io_uring.h

+2 −0
Original line number Diff line number Diff line
@@ -95,6 +95,7 @@ enum { 
#define IORING_SETUP_CQSIZE	(1U << 3)	/* app defines CQ size */

#define IORING_SETUP_CLAMP	(1U << 4)	/* clamp SQ/CQ ring sizes */

#define IORING_SETUP_ATTACH_WQ	(1U << 5)	/* attach to existing wq */

#define IORING_SETUP_R_DISABLED	(1U << 6)	/* start with ring disabled */



enum {

	IORING_OP_NOP,
@@ -268,6 +269,7 @@ enum { 
	IORING_REGISTER_PERSONALITY		= 9,

	IORING_UNREGISTER_PERSONALITY		= 10,

	IORING_REGISTER_RESTRICTIONS		= 11,

	IORING_REGISTER_ENABLE_RINGS		= 12,



	/* this goes last */

	IORING_REGISTER_LAST

CRA Git | Maintained and supported by SUSTech CRA and CCSE