Commit 98d95416 authored by Omar Sandoval's avatar Omar Sandoval Committed by Jens Axboe
Browse files

sbitmap: randomize initial alloc_hint values 



In order to get good cache behavior from a sbitmap, we want each CPU to
stick to its own cacheline(s) as much as possible. This might happen
naturally as the bitmap gets filled up and the alloc_hint values spread
out, but we really want this behavior from the start. blk-mq apparently
intended to do this, but the code to do this was never wired up. Get rid
of the dead code and make it part of the sbitmap library.

Signed-off-by: default avatarOmar Sandoval <osandov@fb.com>
Signed-off-by: default avatarJens Axboe <axboe@fb.com>
parent f4a644db

block/blk-mq-tag.c

+0 −8
Original line number Diff line number Diff line
@@ -7,7 +7,6 @@ 
 */

#include <linux/kernel.h>

#include <linux/module.h>

#include <linux/random.h>



#include <linux/blk-mq.h>

#include "blk.h"
@@ -419,13 +418,6 @@ void blk_mq_free_tags(struct blk_mq_tags *tags) 
	kfree(tags);

}



void blk_mq_tag_init_last_tag(struct blk_mq_tags *tags, unsigned int *tag)

{

	unsigned int depth = tags->nr_tags - tags->nr_reserved_tags;



	*tag = prandom_u32() % depth;

}



int blk_mq_tag_update_depth(struct blk_mq_tags *tags, unsigned int tdepth)

{

	tdepth -= tags->nr_reserved_tags;

block/blk-mq-tag.h

+0 −1
Original line number Diff line number Diff line
@@ -30,7 +30,6 @@ extern void blk_mq_put_tag(struct blk_mq_hw_ctx *hctx, struct blk_mq_ctx *ctx, 
			   unsigned int tag);

extern bool blk_mq_has_free_tags(struct blk_mq_tags *tags);

extern ssize_t blk_mq_tag_sysfs_show(struct blk_mq_tags *tags, char *page);

extern void blk_mq_tag_init_last_tag(struct blk_mq_tags *tags, unsigned int *last_tag);

extern int blk_mq_tag_update_depth(struct blk_mq_tags *tags, unsigned int depth);

extern void blk_mq_tag_wakeup_all(struct blk_mq_tags *tags, bool);

void blk_mq_queue_tag_busy_iter(struct request_queue *q, busy_iter_fn *fn,

lib/sbitmap.c

+6 −0
Original line number Diff line number Diff line
@@ -15,6 +15,7 @@ 
 * along with this program.  If not, see <https://www.gnu.org/licenses/>.

 */



#include <linux/random.h>

#include <linux/sbitmap.h>



int sbitmap_init_node(struct sbitmap *sb, unsigned int depth, int shift,
@@ -211,6 +212,11 @@ int sbitmap_queue_init_node(struct sbitmap_queue *sbq, unsigned int depth, 
		return -ENOMEM;

	}



	if (depth && !round_robin) {

		for_each_possible_cpu(i)

			*per_cpu_ptr(sbq->alloc_hint, i) = prandom_u32() % depth;

	}



	sbq->wake_batch = sbq_calc_wake_batch(depth);

	atomic_set(&sbq->wake_index, 0);

CRA Git | Maintained and supported by SUSTech CRA and CCSE