Commit e8618575 authored by Jens Axboe's avatar Jens Axboe
Browse files

blk-mq: fix sbitmap ws_active for shared tags 



We now wrap sbitmap waitqueues in an active counter, so we can avoid
iterating wakeups unless we have waiters there. This works as long as
everyone that's manipulating the waitqueues use the proper helpers. For
the tag wait case for shared tags, however, we add ourselves to the
waitqueue without incrementing/decrementing the ->ws_active count. This
means that wakeups can take a long time to happen.

Fix this by manually doing the inc/dec as needed for the wait queue
handling.

Reported-by: default avatarMichael Leun <kbug@newton.leun.net>
Tested-by: default avatarMichael Leun <kbug@newton.leun.net>
Cc: stable@vger.kernel.org
Reviewed-by: default avatarOmar Sandoval <osandov@fb.com>
Fixes: 5d2ee712 ("sbitmap: optimize wakeup check")
Signed-off-by: default avatarJens Axboe <axboe@kernel.dk>
parent 9e75ad5d

block/blk-mq.c

+11 −2
Original line number Diff line number Diff line
@@ -1072,7 +1072,13 @@ static int blk_mq_dispatch_wake(wait_queue_entry_t *wait, unsigned mode, 
	hctx = container_of(wait, struct blk_mq_hw_ctx, dispatch_wait);



	spin_lock(&hctx->dispatch_wait_lock);

	if (!list_empty(&wait->entry)) {

		struct sbitmap_queue *sbq;



		list_del_init(&wait->entry);

		sbq = &hctx->tags->bitmap_tags;

		atomic_dec(&sbq->ws_active);

	}

	spin_unlock(&hctx->dispatch_wait_lock);



	blk_mq_run_hw_queue(hctx, true);
@@ -1088,6 +1094,7 @@ static int blk_mq_dispatch_wake(wait_queue_entry_t *wait, unsigned mode, 
static bool blk_mq_mark_tag_wait(struct blk_mq_hw_ctx *hctx,

				 struct request *rq)

{

	struct sbitmap_queue *sbq = &hctx->tags->bitmap_tags;

	struct wait_queue_head *wq;

	wait_queue_entry_t *wait;

	bool ret;
@@ -1110,7 +1117,7 @@ static bool blk_mq_mark_tag_wait(struct blk_mq_hw_ctx *hctx, 
	if (!list_empty_careful(&wait->entry))

		return false;



	wq = &bt_wait_ptr(&hctx->tags->bitmap_tags, hctx)->wait;

	wq = &bt_wait_ptr(sbq, hctx)->wait;



	spin_lock_irq(&wq->lock);

	spin_lock(&hctx->dispatch_wait_lock);
@@ -1120,6 +1127,7 @@ static bool blk_mq_mark_tag_wait(struct blk_mq_hw_ctx *hctx, 
		return false;

	}



	atomic_inc(&sbq->ws_active);

	wait->flags &= ~WQ_FLAG_EXCLUSIVE;

	__add_wait_queue(wq, wait);
@@ -1140,6 +1148,7 @@ static bool blk_mq_mark_tag_wait(struct blk_mq_hw_ctx *hctx, 
	 * someone else gets the wakeup.

	 */

	list_del_init(&wait->entry);

	atomic_dec(&sbq->ws_active);

	spin_unlock(&hctx->dispatch_wait_lock);

	spin_unlock_irq(&wq->lock);

CRA Git | Maintained and supported by SUSTech CRA and CCSE