Commit 8aa6ba2f authored by Kent Overstreet's avatar Kent Overstreet Committed by Jens Axboe
Browse files

block: Convert bio_set to mempool_init() 



Minor performance improvement by getting rid of pointer indirections
from allocation/freeing fastpaths.

Signed-off-by: default avatarKent Overstreet <kent.overstreet@gmail.com>
Signed-off-by: default avatarJens Axboe <axboe@kernel.dk>
parent c1a67fef

block/bio-integrity.c

+14 −15
Original line number Diff line number Diff line
@@ -56,12 +56,12 @@ struct bio_integrity_payload *bio_integrity_alloc(struct bio *bio, 
	struct bio_set *bs = bio->bi_pool;

	unsigned inline_vecs;



	if (!bs || !bs->bio_integrity_pool) {

	if (!bs || !mempool_initialized(&bs->bio_integrity_pool)) {

		bip = kmalloc(sizeof(struct bio_integrity_payload) +

			      sizeof(struct bio_vec) * nr_vecs, gfp_mask);

		inline_vecs = nr_vecs;

	} else {

		bip = mempool_alloc(bs->bio_integrity_pool, gfp_mask);

		bip = mempool_alloc(&bs->bio_integrity_pool, gfp_mask);

		inline_vecs = BIP_INLINE_VECS;

	}
@@ -74,7 +74,7 @@ struct bio_integrity_payload *bio_integrity_alloc(struct bio *bio, 
		unsigned long idx = 0;



		bip->bip_vec = bvec_alloc(gfp_mask, nr_vecs, &idx,

					  bs->bvec_integrity_pool);

					  &bs->bvec_integrity_pool);

		if (!bip->bip_vec)

			goto err;

		bip->bip_max_vcnt = bvec_nr_vecs(idx);
@@ -90,7 +90,7 @@ struct bio_integrity_payload *bio_integrity_alloc(struct bio *bio, 


	return bip;

err:

	mempool_free(bip, bs->bio_integrity_pool);

	mempool_free(bip, &bs->bio_integrity_pool);

	return ERR_PTR(-ENOMEM);

}

EXPORT_SYMBOL(bio_integrity_alloc);
@@ -111,10 +111,10 @@ static void bio_integrity_free(struct bio *bio) 
		kfree(page_address(bip->bip_vec->bv_page) +

		      bip->bip_vec->bv_offset);



	if (bs && bs->bio_integrity_pool) {

		bvec_free(bs->bvec_integrity_pool, bip->bip_vec, bip->bip_slab);

	if (bs && mempool_initialized(&bs->bio_integrity_pool)) {

		bvec_free(&bs->bvec_integrity_pool, bip->bip_vec, bip->bip_slab);



		mempool_free(bip, bs->bio_integrity_pool);

		mempool_free(bip, &bs->bio_integrity_pool);

	} else {

		kfree(bip);

	}
@@ -465,16 +465,15 @@ EXPORT_SYMBOL(bio_integrity_clone); 


int bioset_integrity_create(struct bio_set *bs, int pool_size)

{

	if (bs->bio_integrity_pool)

	if (mempool_initialized(&bs->bio_integrity_pool))

		return 0;



	bs->bio_integrity_pool = mempool_create_slab_pool(pool_size, bip_slab);

	if (!bs->bio_integrity_pool)

	if (mempool_init_slab_pool(&bs->bio_integrity_pool,

				   pool_size, bip_slab))

		return -1;



	bs->bvec_integrity_pool = biovec_create_pool(pool_size);

	if (!bs->bvec_integrity_pool) {

		mempool_destroy(bs->bio_integrity_pool);

	if (biovec_init_pool(&bs->bvec_integrity_pool, pool_size)) {

		mempool_exit(&bs->bio_integrity_pool);

		return -1;

	}
@@ -484,8 +483,8 @@ EXPORT_SYMBOL(bioset_integrity_create); 


void bioset_integrity_free(struct bio_set *bs)

{

	mempool_destroy(bs->bio_integrity_pool);

	mempool_destroy(bs->bvec_integrity_pool);

	mempool_exit(&bs->bio_integrity_pool);

	mempool_exit(&bs->bvec_integrity_pool);

}

EXPORT_SYMBOL(bioset_integrity_free);

block/bio.c

+17 −19
Original line number Diff line number Diff line
@@ -254,7 +254,7 @@ static void bio_free(struct bio *bio) 
	bio_uninit(bio);



	if (bs) {

		bvec_free(bs->bvec_pool, bio->bi_io_vec, BVEC_POOL_IDX(bio));

		bvec_free(&bs->bvec_pool, bio->bi_io_vec, BVEC_POOL_IDX(bio));



		/*

		 * If we have front padding, adjust the bio pointer before freeing
@@ -262,7 +262,7 @@ static void bio_free(struct bio *bio) 
		p = bio;

		p -= bs->front_pad;



		mempool_free(p, bs->bio_pool);

		mempool_free(p, &bs->bio_pool);

	} else {

		/* Bio was allocated by bio_kmalloc() */

		kfree(bio);
@@ -454,7 +454,8 @@ struct bio *bio_alloc_bioset(gfp_t gfp_mask, unsigned int nr_iovecs, 
		inline_vecs = nr_iovecs;

	} else {

		/* should not use nobvec bioset for nr_iovecs > 0 */

		if (WARN_ON_ONCE(!bs->bvec_pool && nr_iovecs > 0))

		if (WARN_ON_ONCE(!mempool_initialized(&bs->bvec_pool) &&

				 nr_iovecs > 0))

			return NULL;

		/*

		 * generic_make_request() converts recursion to iteration; this
@@ -483,11 +484,11 @@ struct bio *bio_alloc_bioset(gfp_t gfp_mask, unsigned int nr_iovecs, 
		    bs->rescue_workqueue)

			gfp_mask &= ~__GFP_DIRECT_RECLAIM;



		p = mempool_alloc(bs->bio_pool, gfp_mask);

		p = mempool_alloc(&bs->bio_pool, gfp_mask);

		if (!p && gfp_mask != saved_gfp) {

			punt_bios_to_rescuer(bs);

			gfp_mask = saved_gfp;

			p = mempool_alloc(bs->bio_pool, gfp_mask);

			p = mempool_alloc(&bs->bio_pool, gfp_mask);

		}



		front_pad = bs->front_pad;
@@ -503,11 +504,11 @@ struct bio *bio_alloc_bioset(gfp_t gfp_mask, unsigned int nr_iovecs, 
	if (nr_iovecs > inline_vecs) {

		unsigned long idx = 0;



		bvl = bvec_alloc(gfp_mask, nr_iovecs, &idx, bs->bvec_pool);

		bvl = bvec_alloc(gfp_mask, nr_iovecs, &idx, &bs->bvec_pool);

		if (!bvl && gfp_mask != saved_gfp) {

			punt_bios_to_rescuer(bs);

			gfp_mask = saved_gfp;

			bvl = bvec_alloc(gfp_mask, nr_iovecs, &idx, bs->bvec_pool);

			bvl = bvec_alloc(gfp_mask, nr_iovecs, &idx, &bs->bvec_pool);

		}



		if (unlikely(!bvl))
@@ -524,7 +525,7 @@ struct bio *bio_alloc_bioset(gfp_t gfp_mask, unsigned int nr_iovecs, 
	return bio;



err_free:

	mempool_free(p, bs->bio_pool);

	mempool_free(p, &bs->bio_pool);

	return NULL;

}

EXPORT_SYMBOL(bio_alloc_bioset);
@@ -1848,11 +1849,11 @@ EXPORT_SYMBOL_GPL(bio_trim); 
 * create memory pools for biovec's in a bio_set.

 * use the global biovec slabs created for general use.

 */

mempool_t *biovec_create_pool(int pool_entries)

int biovec_init_pool(mempool_t *pool, int pool_entries)

{

	struct biovec_slab *bp = bvec_slabs + BVEC_POOL_MAX;



	return mempool_create_slab_pool(pool_entries, bp->slab);

	return mempool_init_slab_pool(pool, pool_entries, bp->slab);

}



void bioset_free(struct bio_set *bs)
@@ -1860,8 +1861,8 @@ void bioset_free(struct bio_set *bs) 
	if (bs->rescue_workqueue)

		destroy_workqueue(bs->rescue_workqueue);



	mempool_destroy(bs->bio_pool);

	mempool_destroy(bs->bvec_pool);

	mempool_exit(&bs->bio_pool);

	mempool_exit(&bs->bvec_pool);



	bioset_integrity_free(bs);

	bio_put_slab(bs);
@@ -1913,15 +1914,12 @@ struct bio_set *bioset_create(unsigned int pool_size, 
		return NULL;

	}



	bs->bio_pool = mempool_create_slab_pool(pool_size, bs->bio_slab);

	if (!bs->bio_pool)

	if (mempool_init_slab_pool(&bs->bio_pool, pool_size, bs->bio_slab))

		goto bad;



	if (flags & BIOSET_NEED_BVECS) {

		bs->bvec_pool = biovec_create_pool(pool_size);

		if (!bs->bvec_pool)

	if ((flags & BIOSET_NEED_BVECS) &&

	    biovec_init_pool(&bs->bvec_pool, pool_size))

		goto bad;

	}



	if (!(flags & BIOSET_NEED_RESCUER))

		return bs;

include/linux/bio.h

+5 −5
Original line number Diff line number Diff line
@@ -412,7 +412,7 @@ enum { 
	BIOSET_NEED_RESCUER = BIT(1),

};

extern void bioset_free(struct bio_set *);

extern mempool_t *biovec_create_pool(int pool_entries);

extern int biovec_init_pool(mempool_t *pool, int pool_entries);



extern struct bio *bio_alloc_bioset(gfp_t, unsigned int, struct bio_set *);

extern void bio_put(struct bio *);
@@ -722,11 +722,11 @@ struct bio_set { 
	struct kmem_cache *bio_slab;

	unsigned int front_pad;



	mempool_t *bio_pool;

	mempool_t *bvec_pool;

	mempool_t bio_pool;

	mempool_t bvec_pool;

#if defined(CONFIG_BLK_DEV_INTEGRITY)

	mempool_t *bio_integrity_pool;

	mempool_t *bvec_integrity_pool;

	mempool_t bio_integrity_pool;

	mempool_t bvec_integrity_pool;

#endif



	/*

CRA Git | Maintained and supported by SUSTech CRA and CCSE