Commit c1a67fef authored by Kent Overstreet's avatar Kent Overstreet Committed by Jens Axboe
Browse files

mempool: Add mempool_init()/mempool_exit() 



Allows mempools to be embedded in other structs, getting rid of a
pointer indirection from allocation fastpaths.

mempool_exit() is safe to call on an uninitialized but zeroed mempool.

Signed-off-by: default avatarKent Overstreet <kent.overstreet@gmail.com>
Signed-off-by: default avatarJens Axboe <axboe@kernel.dk>
parent c854ab57

include/linux/mempool.h

+34 −0
Original line number Diff line number Diff line
@@ -25,6 +25,18 @@ typedef struct mempool_s { 
	wait_queue_head_t wait;

} mempool_t;



static inline bool mempool_initialized(mempool_t *pool)

{

	return pool->elements != NULL;

}



void mempool_exit(mempool_t *pool);

int mempool_init_node(mempool_t *pool, int min_nr, mempool_alloc_t *alloc_fn,

		      mempool_free_t *free_fn, void *pool_data,

		      gfp_t gfp_mask, int node_id);

int mempool_init(mempool_t *pool, int min_nr, mempool_alloc_t *alloc_fn,

		 mempool_free_t *free_fn, void *pool_data);



extern mempool_t *mempool_create(int min_nr, mempool_alloc_t *alloc_fn,

			mempool_free_t *free_fn, void *pool_data);

extern mempool_t *mempool_create_node(int min_nr, mempool_alloc_t *alloc_fn,
@@ -43,6 +55,14 @@ extern void mempool_free(void *element, mempool_t *pool); 
 */

void *mempool_alloc_slab(gfp_t gfp_mask, void *pool_data);

void mempool_free_slab(void *element, void *pool_data);



static inline int

mempool_init_slab_pool(mempool_t *pool, int min_nr, struct kmem_cache *kc)

{

	return mempool_init(pool, min_nr, mempool_alloc_slab,

			    mempool_free_slab, (void *) kc);

}



static inline mempool_t *

mempool_create_slab_pool(int min_nr, struct kmem_cache *kc)

{
@@ -56,6 +76,13 @@ mempool_create_slab_pool(int min_nr, struct kmem_cache *kc) 
 */

void *mempool_kmalloc(gfp_t gfp_mask, void *pool_data);

void mempool_kfree(void *element, void *pool_data);



static inline int mempool_init_kmalloc_pool(mempool_t *pool, int min_nr, size_t size)

{

	return mempool_init(pool, min_nr, mempool_kmalloc,

			    mempool_kfree, (void *) size);

}



static inline mempool_t *mempool_create_kmalloc_pool(int min_nr, size_t size)

{

	return mempool_create(min_nr, mempool_kmalloc, mempool_kfree,
@@ -68,6 +95,13 @@ static inline mempool_t *mempool_create_kmalloc_pool(int min_nr, size_t size) 
 */

void *mempool_alloc_pages(gfp_t gfp_mask, void *pool_data);

void mempool_free_pages(void *element, void *pool_data);



static inline int mempool_init_page_pool(mempool_t *pool, int min_nr, int order)

{

	return mempool_init(pool, min_nr, mempool_alloc_pages,

			    mempool_free_pages, (void *)(long)order);

}



static inline mempool_t *mempool_create_page_pool(int min_nr, int order)

{

	return mempool_create(min_nr, mempool_alloc_pages, mempool_free_pages,

mm/mempool.c

+81 −27
Original line number Diff line number Diff line
@@ -137,6 +137,28 @@ static void *remove_element(mempool_t *pool, gfp_t flags) 
	return element;

}



/**

 * mempool_exit - exit a mempool initialized with mempool_init()

 * @pool:      pointer to the memory pool which was initialized with

 *             mempool_init().

 *

 * Free all reserved elements in @pool and @pool itself.  This function

 * only sleeps if the free_fn() function sleeps.

 *

 * May be called on a zeroed but uninitialized mempool (i.e. allocated with

 * kzalloc()).

 */

void mempool_exit(mempool_t *pool)

{

	while (pool->curr_nr) {

		void *element = remove_element(pool, GFP_KERNEL);

		pool->free(element, pool->pool_data);

	}

	kfree(pool->elements);

	pool->elements = NULL;

}

EXPORT_SYMBOL(mempool_exit);



/**

 * mempool_destroy - deallocate a memory pool

 * @pool:      pointer to the memory pool which was allocated via
@@ -150,15 +172,65 @@ void mempool_destroy(mempool_t *pool) 
	if (unlikely(!pool))

		return;



	while (pool->curr_nr) {

		void *element = remove_element(pool, GFP_KERNEL);

		pool->free(element, pool->pool_data);

	}

	kfree(pool->elements);

	mempool_exit(pool);

	kfree(pool);

}

EXPORT_SYMBOL(mempool_destroy);



int mempool_init_node(mempool_t *pool, int min_nr, mempool_alloc_t *alloc_fn,

		      mempool_free_t *free_fn, void *pool_data,

		      gfp_t gfp_mask, int node_id)

{

	spin_lock_init(&pool->lock);

	pool->min_nr	= min_nr;

	pool->pool_data = pool_data;

	pool->alloc	= alloc_fn;

	pool->free	= free_fn;

	init_waitqueue_head(&pool->wait);



	pool->elements = kmalloc_array_node(min_nr, sizeof(void *),

					    gfp_mask, node_id);

	if (!pool->elements)

		return -ENOMEM;



	/*

	 * First pre-allocate the guaranteed number of buffers.

	 */

	while (pool->curr_nr < pool->min_nr) {

		void *element;



		element = pool->alloc(gfp_mask, pool->pool_data);

		if (unlikely(!element)) {

			mempool_exit(pool);

			return -ENOMEM;

		}

		add_element(pool, element);

	}



	return 0;

}

EXPORT_SYMBOL(mempool_init_node);



/**

 * mempool_init - initialize a memory pool

 * @min_nr:    the minimum number of elements guaranteed to be

 *             allocated for this pool.

 * @alloc_fn:  user-defined element-allocation function.

 * @free_fn:   user-defined element-freeing function.

 * @pool_data: optional private data available to the user-defined functions.

 *

 * Like mempool_create(), but initializes the pool in (i.e. embedded in another

 * structure).

 */

int mempool_init(mempool_t *pool, int min_nr, mempool_alloc_t *alloc_fn,

		 mempool_free_t *free_fn, void *pool_data)

{

	return mempool_init_node(pool, min_nr, alloc_fn, free_fn,

				 pool_data, GFP_KERNEL, NUMA_NO_NODE);



}

EXPORT_SYMBOL(mempool_init);



/**

 * mempool_create - create a memory pool

 * @min_nr:    the minimum number of elements guaranteed to be
@@ -186,35 +258,17 @@ mempool_t *mempool_create_node(int min_nr, mempool_alloc_t *alloc_fn, 
			       gfp_t gfp_mask, int node_id)

{

	mempool_t *pool;



	pool = kzalloc_node(sizeof(*pool), gfp_mask, node_id);

	if (!pool)

		return NULL;

	pool->elements = kmalloc_array_node(min_nr, sizeof(void *),

				      gfp_mask, node_id);

	if (!pool->elements) {



	if (mempool_init_node(pool, min_nr, alloc_fn, free_fn, pool_data,

			      gfp_mask, node_id)) {

		kfree(pool);

		return NULL;

	}

	spin_lock_init(&pool->lock);

	pool->min_nr = min_nr;

	pool->pool_data = pool_data;

	init_waitqueue_head(&pool->wait);

	pool->alloc = alloc_fn;

	pool->free = free_fn;



	/*

	 * First pre-allocate the guaranteed number of buffers.

	 */

	while (pool->curr_nr < pool->min_nr) {

		void *element;



		element = pool->alloc(gfp_mask, pool->pool_data);

		if (unlikely(!element)) {

			mempool_destroy(pool);

			return NULL;

		}

		add_element(pool, element);

	}

	return pool;

}

EXPORT_SYMBOL(mempool_create_node);

CRA Git | Maintained and supported by SUSTech CRA and CCSE