Commit bff157b3 authored by Linus Torvalds's avatar Linus Torvalds
Browse files

Merge branch 'slab/next' of git://git.kernel.org/pub/scm/linux/kernel/git/penberg/linux 

Pull SLAB update from Pekka Enberg:
 "Nothing terribly exciting here apart from Christoph's kmalloc
  unification patches that brings sl[aou]b implementations closer to
  each other"

* 'slab/next' of git://git.kernel.org/pub/scm/linux/kernel/git/penberg/linux:
  slab: Use correct GFP_DMA constant
  slub: remove verify_mem_not_deleted()
  mm/sl[aou]b: Move kmallocXXX functions to common code
  mm, slab_common: add 'unlikely' to size check of kmalloc_slab()
  mm/slub.c: beautify code for removing redundancy 'break' statement.
  slub: Remove unnecessary page NULL check
  slub: don't use cpu partial pages on UP
  mm/slub: beautify code for 80 column limitation and tab alignment
  mm/slub: remove 'per_cpu' which is useless variable
parents 8bf5e36d 23774a2f

include/linux/slab.h

+124 −32
Original line number Diff line number Diff line
@@ -4,6 +4,8 @@ 
 * (C) SGI 2006, Christoph Lameter

 * 	Cleaned up and restructured to ease the addition of alternative

 * 	implementations of SLAB allocators.

 * (C) Linux Foundation 2008-2013

 *      Unified interface for all slab allocators

 */



#ifndef _LINUX_SLAB_H
@@ -94,6 +96,7 @@ 
#define ZERO_OR_NULL_PTR(x) ((unsigned long)(x) <= \

				(unsigned long)ZERO_SIZE_PTR)



#include <linux/kmemleak.h>



struct mem_cgroup;

/*
@@ -289,6 +292,57 @@ static __always_inline int kmalloc_index(size_t size) 
}

#endif /* !CONFIG_SLOB */



void *__kmalloc(size_t size, gfp_t flags);

void *kmem_cache_alloc(struct kmem_cache *, gfp_t flags);



#ifdef CONFIG_NUMA

void *__kmalloc_node(size_t size, gfp_t flags, int node);

void *kmem_cache_alloc_node(struct kmem_cache *, gfp_t flags, int node);

#else

static __always_inline void *__kmalloc_node(size_t size, gfp_t flags, int node)

{

	return __kmalloc(size, flags);

}



static __always_inline void *kmem_cache_alloc_node(struct kmem_cache *s, gfp_t flags, int node)

{

	return kmem_cache_alloc(s, flags);

}

#endif



#ifdef CONFIG_TRACING

extern void *kmem_cache_alloc_trace(struct kmem_cache *, gfp_t, size_t);



#ifdef CONFIG_NUMA

extern void *kmem_cache_alloc_node_trace(struct kmem_cache *s,

					   gfp_t gfpflags,

					   int node, size_t size);

#else

static __always_inline void *

kmem_cache_alloc_node_trace(struct kmem_cache *s,

			      gfp_t gfpflags,

			      int node, size_t size)

{

	return kmem_cache_alloc_trace(s, gfpflags, size);

}

#endif /* CONFIG_NUMA */



#else /* CONFIG_TRACING */

static __always_inline void *kmem_cache_alloc_trace(struct kmem_cache *s,

		gfp_t flags, size_t size)

{

	return kmem_cache_alloc(s, flags);

}



static __always_inline void *

kmem_cache_alloc_node_trace(struct kmem_cache *s,

			      gfp_t gfpflags,

			      int node, size_t size)

{

	return kmem_cache_alloc_node(s, gfpflags, node);

}

#endif /* CONFIG_TRACING */



#ifdef CONFIG_SLAB

#include <linux/slab_def.h>

#endif
@@ -297,10 +351,61 @@ static __always_inline int kmalloc_index(size_t size) 
#include <linux/slub_def.h>

#endif



#ifdef CONFIG_SLOB

#include <linux/slob_def.h>

static __always_inline void *

kmalloc_order(size_t size, gfp_t flags, unsigned int order)

{

	void *ret;



	flags |= (__GFP_COMP | __GFP_KMEMCG);

	ret = (void *) __get_free_pages(flags, order);

	kmemleak_alloc(ret, size, 1, flags);

	return ret;

}



#ifdef CONFIG_TRACING

extern void *kmalloc_order_trace(size_t size, gfp_t flags, unsigned int order);

#else

static __always_inline void *

kmalloc_order_trace(size_t size, gfp_t flags, unsigned int order)

{

	return kmalloc_order(size, flags, order);

}

#endif



static __always_inline void *kmalloc_large(size_t size, gfp_t flags)

{

	unsigned int order = get_order(size);

	return kmalloc_order_trace(size, flags, order);

}



/**

 * kmalloc - allocate memory

 * @size: how many bytes of memory are required.

 * @flags: the type of memory to allocate (see kcalloc).

 *

 * kmalloc is the normal method of allocating memory

 * for objects smaller than page size in the kernel.

 */

static __always_inline void *kmalloc(size_t size, gfp_t flags)

{

	if (__builtin_constant_p(size)) {

		if (size > KMALLOC_MAX_CACHE_SIZE)

			return kmalloc_large(size, flags);

#ifndef CONFIG_SLOB

		if (!(flags & GFP_DMA)) {

			int index = kmalloc_index(size);



			if (!index)

				return ZERO_SIZE_PTR;



			return kmem_cache_alloc_trace(kmalloc_caches[index],

					flags, size);

		}

#endif

	}

	return __kmalloc(size, flags);

}



/*

 * Determine size used for the nth kmalloc cache.

 * return size or 0 if a kmalloc cache for that
@@ -321,6 +426,23 @@ static __always_inline int kmalloc_size(int n) 
	return 0;

}



static __always_inline void *kmalloc_node(size_t size, gfp_t flags, int node)

{

#ifndef CONFIG_SLOB

	if (__builtin_constant_p(size) &&

		size <= KMALLOC_MAX_CACHE_SIZE && !(flags & GFP_DMA)) {

		int i = kmalloc_index(size);



		if (!i)

			return ZERO_SIZE_PTR;



		return kmem_cache_alloc_node_trace(kmalloc_caches[i],

						flags, node, size);

	}

#endif

	return __kmalloc_node(size, flags, node);

}



/*

 * Setting ARCH_SLAB_MINALIGN in arch headers allows a different alignment.

 * Intended for arches that get misalignment faults even for 64 bit integer
@@ -451,36 +573,6 @@ static inline void *kcalloc(size_t n, size_t size, gfp_t flags) 
	return kmalloc_array(n, size, flags | __GFP_ZERO);

}



#if !defined(CONFIG_NUMA) && !defined(CONFIG_SLOB)

/**

 * kmalloc_node - allocate memory from a specific node

 * @size: how many bytes of memory are required.

 * @flags: the type of memory to allocate (see kmalloc).

 * @node: node to allocate from.

 *

 * kmalloc() for non-local nodes, used to allocate from a specific node

 * if available. Equivalent to kmalloc() in the non-NUMA single-node

 * case.

 */

static inline void *kmalloc_node(size_t size, gfp_t flags, int node)

{

	return kmalloc(size, flags);

}



static inline void *__kmalloc_node(size_t size, gfp_t flags, int node)

{

	return __kmalloc(size, flags);

}



void *kmem_cache_alloc(struct kmem_cache *, gfp_t);



static inline void *kmem_cache_alloc_node(struct kmem_cache *cachep,

					gfp_t flags, int node)

{

	return kmem_cache_alloc(cachep, flags);

}

#endif /* !CONFIG_NUMA && !CONFIG_SLOB */



/*

 * kmalloc_track_caller is a special version of kmalloc that records the

 * calling function of the routine calling it for slab leak tracking instead

include/linux/slab_def.h

+0 −106
Original line number Diff line number Diff line
@@ -3,20 +3,6 @@ 


/*

 * Definitions unique to the original Linux SLAB allocator.

 *

 * What we provide here is a way to optimize the frequent kmalloc

 * calls in the kernel by selecting the appropriate general cache

 * if kmalloc was called with a size that can be established at

 * compile time.

 */



#include <linux/init.h>

#include <linux/compiler.h>



/*

 * struct kmem_cache

 *

 * manages a cache.

 */



struct kmem_cache {
@@ -102,96 +88,4 @@ struct kmem_cache { 
	 */

};



void *kmem_cache_alloc(struct kmem_cache *, gfp_t);

void *__kmalloc(size_t size, gfp_t flags);



#ifdef CONFIG_TRACING

extern void *kmem_cache_alloc_trace(struct kmem_cache *, gfp_t, size_t);

#else

static __always_inline void *

kmem_cache_alloc_trace(struct kmem_cache *cachep, gfp_t flags, size_t size)

{

	return kmem_cache_alloc(cachep, flags);

}

#endif



static __always_inline void *kmalloc(size_t size, gfp_t flags)

{

	struct kmem_cache *cachep;

	void *ret;



	if (__builtin_constant_p(size)) {

		int i;



		if (!size)

			return ZERO_SIZE_PTR;



		if (WARN_ON_ONCE(size > KMALLOC_MAX_SIZE))

			return NULL;



		i = kmalloc_index(size);



#ifdef CONFIG_ZONE_DMA

		if (flags & GFP_DMA)

			cachep = kmalloc_dma_caches[i];

		else

#endif

			cachep = kmalloc_caches[i];



		ret = kmem_cache_alloc_trace(cachep, flags, size);



		return ret;

	}

	return __kmalloc(size, flags);

}



#ifdef CONFIG_NUMA

extern void *__kmalloc_node(size_t size, gfp_t flags, int node);

extern void *kmem_cache_alloc_node(struct kmem_cache *, gfp_t flags, int node);



#ifdef CONFIG_TRACING

extern void *kmem_cache_alloc_node_trace(struct kmem_cache *cachep,

					 gfp_t flags,

					 int nodeid,

					 size_t size);

#else

static __always_inline void *

kmem_cache_alloc_node_trace(struct kmem_cache *cachep,

			    gfp_t flags,

			    int nodeid,

			    size_t size)

{

	return kmem_cache_alloc_node(cachep, flags, nodeid);

}

#endif



static __always_inline void *kmalloc_node(size_t size, gfp_t flags, int node)

{

	struct kmem_cache *cachep;



	if (__builtin_constant_p(size)) {

		int i;



		if (!size)

			return ZERO_SIZE_PTR;



		if (WARN_ON_ONCE(size > KMALLOC_MAX_SIZE))

			return NULL;



		i = kmalloc_index(size);



#ifdef CONFIG_ZONE_DMA

		if (flags & GFP_DMA)

			cachep = kmalloc_dma_caches[i];

		else

#endif

			cachep = kmalloc_caches[i];



		return kmem_cache_alloc_node_trace(cachep, flags, node, size);

	}

	return __kmalloc_node(size, flags, node);

}



#endif	/* CONFIG_NUMA */



#endif	/* _LINUX_SLAB_DEF_H */

include/linux/slob_def.h

deleted100644 → 0
+0 −31
Original line number Diff line number Diff line 
#ifndef __LINUX_SLOB_DEF_H

#define __LINUX_SLOB_DEF_H



#include <linux/numa.h>



void *kmem_cache_alloc_node(struct kmem_cache *, gfp_t flags, int node);



static __always_inline void *kmem_cache_alloc(struct kmem_cache *cachep,

					      gfp_t flags)

{

	return kmem_cache_alloc_node(cachep, flags, NUMA_NO_NODE);

}



void *__kmalloc_node(size_t size, gfp_t flags, int node);



static __always_inline void *kmalloc_node(size_t size, gfp_t flags, int node)

{

	return __kmalloc_node(size, flags, node);

}



static __always_inline void *kmalloc(size_t size, gfp_t flags)

{

	return __kmalloc_node(size, flags, NUMA_NO_NODE);

}



static __always_inline void *__kmalloc(size_t size, gfp_t flags)

{

	return kmalloc(size, flags);

}



#endif /* __LINUX_SLOB_DEF_H */

include/linux/slub_def.h

+0 −110
Original line number Diff line number Diff line
@@ -6,14 +6,8 @@ 
 *

 * (C) 2007 SGI, Christoph Lameter

 */

#include <linux/types.h>

#include <linux/gfp.h>

#include <linux/bug.h>

#include <linux/workqueue.h>

#include <linux/kobject.h>



#include <linux/kmemleak.h>



enum stat_item {

	ALLOC_FASTPATH,		/* Allocation from cpu slab */

	ALLOC_SLOWPATH,		/* Allocation by getting a new cpu slab */
@@ -104,108 +98,4 @@ struct kmem_cache { 
	struct kmem_cache_node *node[MAX_NUMNODES];

};



void *kmem_cache_alloc(struct kmem_cache *, gfp_t);

void *__kmalloc(size_t size, gfp_t flags);



static __always_inline void *

kmalloc_order(size_t size, gfp_t flags, unsigned int order)

{

	void *ret;



	flags |= (__GFP_COMP | __GFP_KMEMCG);

	ret = (void *) __get_free_pages(flags, order);

	kmemleak_alloc(ret, size, 1, flags);

	return ret;

}



/**

 * Calling this on allocated memory will check that the memory

 * is expected to be in use, and print warnings if not.

 */

#ifdef CONFIG_SLUB_DEBUG

extern bool verify_mem_not_deleted(const void *x);

#else

static inline bool verify_mem_not_deleted(const void *x)

{

	return true;

}

#endif



#ifdef CONFIG_TRACING

extern void *

kmem_cache_alloc_trace(struct kmem_cache *s, gfp_t gfpflags, size_t size);

extern void *kmalloc_order_trace(size_t size, gfp_t flags, unsigned int order);

#else

static __always_inline void *

kmem_cache_alloc_trace(struct kmem_cache *s, gfp_t gfpflags, size_t size)

{

	return kmem_cache_alloc(s, gfpflags);

}



static __always_inline void *

kmalloc_order_trace(size_t size, gfp_t flags, unsigned int order)

{

	return kmalloc_order(size, flags, order);

}

#endif



static __always_inline void *kmalloc_large(size_t size, gfp_t flags)

{

	unsigned int order = get_order(size);

	return kmalloc_order_trace(size, flags, order);

}



static __always_inline void *kmalloc(size_t size, gfp_t flags)

{

	if (__builtin_constant_p(size)) {

		if (size > KMALLOC_MAX_CACHE_SIZE)

			return kmalloc_large(size, flags);



		if (!(flags & GFP_DMA)) {

			int index = kmalloc_index(size);



			if (!index)

				return ZERO_SIZE_PTR;



			return kmem_cache_alloc_trace(kmalloc_caches[index],

					flags, size);

		}

	}

	return __kmalloc(size, flags);

}



#ifdef CONFIG_NUMA

void *__kmalloc_node(size_t size, gfp_t flags, int node);

void *kmem_cache_alloc_node(struct kmem_cache *, gfp_t flags, int node);



#ifdef CONFIG_TRACING

extern void *kmem_cache_alloc_node_trace(struct kmem_cache *s,

					   gfp_t gfpflags,

					   int node, size_t size);

#else

static __always_inline void *

kmem_cache_alloc_node_trace(struct kmem_cache *s,

			      gfp_t gfpflags,

			      int node, size_t size)

{

	return kmem_cache_alloc_node(s, gfpflags, node);

}

#endif



static __always_inline void *kmalloc_node(size_t size, gfp_t flags, int node)

{

	if (__builtin_constant_p(size) &&

		size <= KMALLOC_MAX_CACHE_SIZE && !(flags & GFP_DMA)) {

		int index = kmalloc_index(size);



		if (!index)

			return ZERO_SIZE_PTR;



		return kmem_cache_alloc_node_trace(kmalloc_caches[index],

			       flags, node, size);

	}

	return __kmalloc_node(size, flags, node);

}

#endif



#endif /* _LINUX_SLUB_DEF_H */

init/Kconfig

+1 −1
Original line number Diff line number Diff line
@@ -1602,7 +1602,7 @@ endchoice 


config SLUB_CPU_PARTIAL

	default y

	depends on SLUB

	depends on SLUB && SMP

	bool "SLUB per cpu partial cache"

	help

	  Per cpu partial caches accellerate objects allocation and freeing

CRA Git | Maintained and supported by SUSTech CRA and CCSE