Commit 73367bd8 authored by Alexander Duyck's avatar Alexander Duyck Committed by Pekka Enberg
Browse files

slub: move kmem_cache_node into it's own cacheline 

This patch is meant to improve the performance of SLUB by moving the local
kmem_cache_node lock into it's own cacheline separate from kmem_cache.
This is accomplished by simply removing the local_node when NUMA is enabled.

On my system with 2 nodes I saw around a 5% performance increase w/
hackbench times dropping from 6.2 seconds to 5.9 seconds on average.  I
suspect the performance gain would increase as the number of nodes
increases, but I do not have the data to currently back that up.

Bugzilla-Reference: http://bugzilla.kernel.org/show_bug.cgi?id=15713


Cc: <stable@kernel.org>
Reported-by: default avatarAlex Shi <alex.shi@intel.com>
Tested-by: default avatarAlex Shi <alex.shi@intel.com>
Acked-by: default avatarYanmin Zhang <yanmin_zhang@linux.intel.com>
Acked-by: default avatarChristoph Lameter <cl@linux-foundation.org>
Signed-off-by: default avatarAlexander Duyck <alexander.h.duyck@intel.com>
Signed-off-by: default avatarPekka Enberg <penberg@cs.helsinki.fi>
parent 7e125f7b

include/linux/slub_def.h

+3 −6
Original line number Diff line number Diff line
@@ -75,12 +75,6 @@ struct kmem_cache { 
	int offset;		/* Free pointer offset. */

	struct kmem_cache_order_objects oo;



	/*

	 * Avoid an extra cache line for UP, SMP and for the node local to

	 * struct kmem_cache.

	 */

	struct kmem_cache_node local_node;



	/* Allocation and freeing of slabs */

	struct kmem_cache_order_objects max;

	struct kmem_cache_order_objects min;
@@ -102,6 +96,9 @@ struct kmem_cache { 
	 */

	int remote_node_defrag_ratio;

	struct kmem_cache_node *node[MAX_NUMNODES];

#else

	/* Avoid an extra cache line for UP */

	struct kmem_cache_node local_node;

#endif

};

mm/slub.c

+11 −22
Original line number Diff line number Diff line
@@ -2133,7 +2133,7 @@ static void free_kmem_cache_nodes(struct kmem_cache *s) 


	for_each_node_state(node, N_NORMAL_MEMORY) {

		struct kmem_cache_node *n = s->node[node];

		if (n && n != &s->local_node)

		if (n)

			kmem_cache_free(kmalloc_caches, n);

		s->node[node] = NULL;

	}
@@ -2142,20 +2142,10 @@ static void free_kmem_cache_nodes(struct kmem_cache *s) 
static int init_kmem_cache_nodes(struct kmem_cache *s, gfp_t gfpflags)

{

	int node;

	int local_node;



	if (slab_state >= UP && (s < kmalloc_caches ||

			s >= kmalloc_caches + KMALLOC_CACHES))

		local_node = page_to_nid(virt_to_page(s));

	else

		local_node = 0;



	for_each_node_state(node, N_NORMAL_MEMORY) {

		struct kmem_cache_node *n;



		if (local_node == node)

			n = &s->local_node;

		else {

		if (slab_state == DOWN) {

			early_kmem_cache_node_alloc(gfpflags, node);

			continue;
@@ -2168,7 +2158,6 @@ static int init_kmem_cache_nodes(struct kmem_cache *s, gfp_t gfpflags) 
			return 0;

		}



		}

		s->node[node] = n;

		init_kmem_cache_node(n, s);

	}

CRA Git | Maintained and supported by SUSTech CRA and CCSE