Commit 24369c2e authored by Pavel Begunkov's avatar Pavel Begunkov Committed by Jens Axboe
Browse files

io_uring: add io-wq workqueue sharing 



If IORING_SETUP_ATTACH_WQ is set, it expects wq_fd in io_uring_params to
be a valid io_uring fd io-wq of which will be shared with the newly
created io_uring instance. If the flag is set but it can't share io-wq,
it fails.

This allows creation of "sibling" io_urings, where we prefer to keep the
SQ/CQ private, but want to share the async backend to minimize the amount
of overhead associated with having multiple rings that belong to the same
backend.

Reported-by: default avatarJens Axboe <axboe@kernel.dk>
Reported-by: default avatarDaurnimator <quae@daurnimator.com>
Signed-off-by: default avatarPavel Begunkov <asml.silence@gmail.com>
Signed-off-by: default avatarJens Axboe <axboe@kernel.dk>
parent eba6f5a3

fs/io_uring.c

+50 −14
Original line number Original line Diff line number Diff line
@@ -5704,11 +5704,56 @@ static void io_get_work(struct io_wq_work *work) 
	refcount_inc(&req->refs);

	refcount_inc(&req->refs);

}

}





static int io_sq_offload_start(struct io_ring_ctx *ctx,

static int io_init_wq_offload(struct io_ring_ctx *ctx,

			      struct io_uring_params *p)

			      struct io_uring_params *p)

{

{

	struct io_wq_data data;

	struct io_wq_data data;

	unsigned concurrency;

	struct fd f;

	struct io_ring_ctx *ctx_attach;

	unsigned int concurrency;

	int ret = 0;



	data.user = ctx->user;

	data.get_work = io_get_work;

	data.put_work = io_put_work;



	if (!(p->flags & IORING_SETUP_ATTACH_WQ)) {

		/* Do QD, or 4 * CPUS, whatever is smallest */

		concurrency = min(ctx->sq_entries, 4 * num_online_cpus());



		ctx->io_wq = io_wq_create(concurrency, &data);

		if (IS_ERR(ctx->io_wq)) {

			ret = PTR_ERR(ctx->io_wq);

			ctx->io_wq = NULL;

		}

		return ret;

	}



	f = fdget(p->wq_fd);

	if (!f.file)

		return -EBADF;



	if (f.file->f_op != &io_uring_fops) {

		ret = -EINVAL;

		goto out_fput;

	}



	ctx_attach = f.file->private_data;

	/* @io_wq is protected by holding the fd */

	if (!io_wq_get(ctx_attach->io_wq, &data)) {

		ret = -EINVAL;

		goto out_fput;

	}



	ctx->io_wq = ctx_attach->io_wq;

out_fput:

	fdput(f);

	return ret;

}



static int io_sq_offload_start(struct io_ring_ctx *ctx,

			       struct io_uring_params *p)

{

	int ret;

	int ret;





	init_waitqueue_head(&ctx->sqo_wait);

	init_waitqueue_head(&ctx->sqo_wait);
@@ -5752,18 +5797,9 @@ static int io_sq_offload_start(struct io_ring_ctx *ctx, 
		goto err;

		goto err;

	}

	}





	data.user = ctx->user;

	ret = io_init_wq_offload(ctx, p);

	data.get_work = io_get_work;

	if (ret)

	data.put_work = io_put_work;



	/* Do QD, or 4 * CPUS, whatever is smallest */

	concurrency = min(ctx->sq_entries, 4 * num_online_cpus());

	ctx->io_wq = io_wq_create(concurrency, &data);

	if (IS_ERR(ctx->io_wq)) {

		ret = PTR_ERR(ctx->io_wq);

		ctx->io_wq = NULL;

		goto err;

		goto err;

	}





	return 0;

	return 0;

err:

err:
@@ -6589,7 +6625,7 @@ static long io_uring_setup(u32 entries, struct io_uring_params __user *params) 




	if (p.flags & ~(IORING_SETUP_IOPOLL | IORING_SETUP_SQPOLL |

	if (p.flags & ~(IORING_SETUP_IOPOLL | IORING_SETUP_SQPOLL |

			IORING_SETUP_SQ_AFF | IORING_SETUP_CQSIZE |

			IORING_SETUP_SQ_AFF | IORING_SETUP_CQSIZE |

			IORING_SETUP_CLAMP))

			IORING_SETUP_CLAMP | IORING_SETUP_ATTACH_WQ))

		return -EINVAL;

		return -EINVAL;





	ret = io_uring_create(entries, &p);

	ret = io_uring_create(entries, &p);

include/uapi/linux/io_uring.h

+3 −1
Original line number Original line Diff line number Diff line
@@ -75,6 +75,7 @@ enum { 
#define IORING_SETUP_SQ_AFF	(1U << 2)	/* sq_thread_cpu is valid */

#define IORING_SETUP_SQ_AFF	(1U << 2)	/* sq_thread_cpu is valid */

#define IORING_SETUP_CQSIZE	(1U << 3)	/* app defines CQ size */

#define IORING_SETUP_CQSIZE	(1U << 3)	/* app defines CQ size */

#define IORING_SETUP_CLAMP	(1U << 4)	/* clamp SQ/CQ ring sizes */

#define IORING_SETUP_CLAMP	(1U << 4)	/* clamp SQ/CQ ring sizes */

#define IORING_SETUP_ATTACH_WQ	(1U << 5)	/* attach to existing wq */





enum {

enum {

	IORING_OP_NOP,

	IORING_OP_NOP,
@@ -183,7 +184,8 @@ struct io_uring_params { 
	__u32 sq_thread_cpu;

	__u32 sq_thread_cpu;

	__u32 sq_thread_idle;

	__u32 sq_thread_idle;

	__u32 features;

	__u32 features;

	__u32 resv[4];

	__u32 wq_fd;

	__u32 resv[3];

	struct io_sqring_offsets sq_off;

	struct io_sqring_offsets sq_off;

	struct io_cqring_offsets cq_off;

	struct io_cqring_offsets cq_off;

};

};

CRA Git | Maintained and supported by SUSTech CRA and CCSE