Commit 570e9b73 authored by Ming Lei's avatar Ming Lei Committed by Jens Axboe
Browse files

blk-mq: move blk_mq_get_driver_tag into blk-mq.c 



blk_mq_get_driver_tag() is only used by blk-mq.c and is supposed to
stay in blk-mq.c, so move it and preparing for cleanup code of
get/put driver tag.

Meantime hctx_may_queue() is moved to header file and it is fine
since it is defined as inline always.

No functional change.

Signed-off-by: default avatarMing Lei <ming.lei@redhat.com>
Reviewed-by: default avatarChristoph Hellwig <hch@lst.de>
Reviewed-by: default avatarHannes Reinecke <hare@suse.de>
Cc: Christoph Hellwig <hch@infradead.org>
Signed-off-by: default avatarJens Axboe <axboe@kernel.dk>
parent 6e6fcbc2

block/blk-mq-tag.c

+0 −58
Original line number Diff line number Diff line
@@ -56,37 +56,6 @@ void __blk_mq_tag_idle(struct blk_mq_hw_ctx *hctx) 
	blk_mq_tag_wakeup_all(tags, false);

}



/*

 * For shared tag users, we track the number of currently active users

 * and attempt to provide a fair share of the tag depth for each of them.

 */

static inline bool hctx_may_queue(struct blk_mq_hw_ctx *hctx,

				  struct sbitmap_queue *bt)

{

	unsigned int depth, users;



	if (!hctx || !(hctx->flags & BLK_MQ_F_TAG_SHARED))

		return true;

	if (!test_bit(BLK_MQ_S_TAG_ACTIVE, &hctx->state))

		return true;



	/*

	 * Don't try dividing an ant

	 */

	if (bt->sb.depth == 1)

		return true;



	users = atomic_read(&hctx->tags->active_queues);

	if (!users)

		return true;



	/*

	 * Allow at least some tags

	 */

	depth = max((bt->sb.depth + users - 1) / users, 4U);

	return atomic_read(&hctx->nr_active) < depth;

}



static int __blk_mq_get_tag(struct blk_mq_alloc_data *data,

			    struct sbitmap_queue *bt)

{
@@ -191,33 +160,6 @@ found_tag: 
	return tag + tag_offset;

}



bool __blk_mq_get_driver_tag(struct request *rq)

{

	struct sbitmap_queue *bt = &rq->mq_hctx->tags->bitmap_tags;

	unsigned int tag_offset = rq->mq_hctx->tags->nr_reserved_tags;

	bool shared = blk_mq_tag_busy(rq->mq_hctx);

	int tag;



	if (blk_mq_tag_is_reserved(rq->mq_hctx->sched_tags, rq->internal_tag)) {

		bt = &rq->mq_hctx->tags->breserved_tags;

		tag_offset = 0;

	}



	if (!hctx_may_queue(rq->mq_hctx, bt))

		return false;

	tag = __sbitmap_queue_get(bt);

	if (tag == BLK_MQ_NO_TAG)

		return false;



	rq->tag = tag + tag_offset;

	if (shared) {

		rq->rq_flags |= RQF_MQ_INFLIGHT;

		atomic_inc(&rq->mq_hctx->nr_active);

	}

	rq->mq_hctx->tags->rqs[rq->tag] = rq;

	return true;

}



void blk_mq_put_tag(struct blk_mq_tags *tags, struct blk_mq_ctx *ctx,

		    unsigned int tag)

{

block/blk-mq-tag.h

+31 −8
Original line number Diff line number Diff line
@@ -51,14 +51,6 @@ enum { 
	BLK_MQ_TAG_MAX		= BLK_MQ_NO_TAG - 1,

};



bool __blk_mq_get_driver_tag(struct request *rq);

static inline bool blk_mq_get_driver_tag(struct request *rq)

{

	if (rq->tag != BLK_MQ_NO_TAG)

		return true;

	return __blk_mq_get_driver_tag(rq);

}



extern bool __blk_mq_tag_busy(struct blk_mq_hw_ctx *);

extern void __blk_mq_tag_idle(struct blk_mq_hw_ctx *);
@@ -78,6 +70,37 @@ static inline void blk_mq_tag_idle(struct blk_mq_hw_ctx *hctx) 
	__blk_mq_tag_idle(hctx);

}



/*

 * For shared tag users, we track the number of currently active users

 * and attempt to provide a fair share of the tag depth for each of them.

 */

static inline bool hctx_may_queue(struct blk_mq_hw_ctx *hctx,

				  struct sbitmap_queue *bt)

{

	unsigned int depth, users;



	if (!hctx || !(hctx->flags & BLK_MQ_F_TAG_SHARED))

		return true;

	if (!test_bit(BLK_MQ_S_TAG_ACTIVE, &hctx->state))

		return true;



	/*

	 * Don't try dividing an ant

	 */

	if (bt->sb.depth == 1)

		return true;



	users = atomic_read(&hctx->tags->active_queues);

	if (!users)

		return true;



	/*

	 * Allow at least some tags

	 */

	depth = max((bt->sb.depth + users - 1) / users, 4U);

	return atomic_read(&hctx->nr_active) < depth;

}



/*

 * This helper should only be used for flush request to share tag

 * with the request cloned from, and both the two requests can't be

block/blk-mq.c

+34 −0
Original line number Diff line number Diff line
@@ -1103,6 +1103,40 @@ static inline unsigned int queued_to_index(unsigned int queued) 
	return min(BLK_MQ_MAX_DISPATCH_ORDER - 1, ilog2(queued) + 1);

}



static bool __blk_mq_get_driver_tag(struct request *rq)

{

	struct sbitmap_queue *bt = &rq->mq_hctx->tags->bitmap_tags;

	unsigned int tag_offset = rq->mq_hctx->tags->nr_reserved_tags;

	bool shared = blk_mq_tag_busy(rq->mq_hctx);

	int tag;



	if (blk_mq_tag_is_reserved(rq->mq_hctx->sched_tags, rq->internal_tag)) {

		bt = &rq->mq_hctx->tags->breserved_tags;

		tag_offset = 0;

	}



	if (!hctx_may_queue(rq->mq_hctx, bt))

		return false;

	tag = __sbitmap_queue_get(bt);

	if (tag == BLK_MQ_NO_TAG)

		return false;



	rq->tag = tag + tag_offset;

	if (shared) {

		rq->rq_flags |= RQF_MQ_INFLIGHT;

		atomic_inc(&rq->mq_hctx->nr_active);

	}

	rq->mq_hctx->tags->rqs[rq->tag] = rq;

	return true;

}



static bool blk_mq_get_driver_tag(struct request *rq)

{

	if (rq->tag != BLK_MQ_NO_TAG)

		return true;

	return __blk_mq_get_driver_tag(rq);

}



static int blk_mq_dispatch_wake(wait_queue_entry_t *wait, unsigned mode,

				int flags, void *key)

{

CRA Git | Maintained and supported by SUSTech CRA and CCSE