Commit f99b7880 authored by Linus Torvalds's avatar Linus Torvalds
Browse files

Merge branch 'slab-for-linus' of git://git.kernel.org/pub/scm/linux/kernel/git/penberg/slab-2.6 

* 'slab-for-linus' of git://git.kernel.org/pub/scm/linux/kernel/git/penberg/slab-2.6:
  slab: fix DEBUG_SLAB warning
  slab: shrink sizeof(struct kmem_cache)
  slab: fix DEBUG_SLAB build
  SLUB: Fix missing <linux/stacktrace.h> include
  slub: reduce overhead of slub_debug
  slub: Add method to verify memory is not freed
  slub: Enable backtrace for create/delete points
  slab allocators: Provide generic description of alignment defines
  slab, slub, slob: Unify alignment definition
  slob/lockdep: Fix gfp flags passed to lockdep
parents 02f8c6ae 7ea466f2

include/linux/slab.h

+20 −0
Original line number Diff line number Diff line
@@ -133,6 +133,26 @@ unsigned int kmem_cache_size(struct kmem_cache *); 
#define KMALLOC_MAX_SIZE	(1UL << KMALLOC_SHIFT_HIGH)

#define KMALLOC_MAX_ORDER	(KMALLOC_SHIFT_HIGH - PAGE_SHIFT)



/*

 * Some archs want to perform DMA into kmalloc caches and need a guaranteed

 * alignment larger than the alignment of a 64-bit integer.

 * Setting ARCH_KMALLOC_MINALIGN in arch headers allows that.

 */

#ifdef ARCH_DMA_MINALIGN

#define ARCH_KMALLOC_MINALIGN ARCH_DMA_MINALIGN

#else

#define ARCH_KMALLOC_MINALIGN __alignof__(unsigned long long)

#endif



/*

 * Setting ARCH_SLAB_MINALIGN in arch headers allows a different alignment.

 * Intended for arches that get misalignment faults even for 64 bit integer

 * aligned buffers.

 */

#ifndef ARCH_SLAB_MINALIGN

#define ARCH_SLAB_MINALIGN __alignof__(unsigned long long)

#endif



/*

 * Common kmalloc functions provided by all allocators

 */

include/linux/slab_def.h

+13 −39
Original line number Diff line number Diff line
@@ -17,32 +17,6 @@ 


#include <trace/events/kmem.h>



/*

 * Enforce a minimum alignment for the kmalloc caches.

 * Usually, the kmalloc caches are cache_line_size() aligned, except when

 * DEBUG and FORCED_DEBUG are enabled, then they are BYTES_PER_WORD aligned.

 * Some archs want to perform DMA into kmalloc caches and need a guaranteed

 * alignment larger than the alignment of a 64-bit integer.

 * ARCH_KMALLOC_MINALIGN allows that.

 * Note that increasing this value may disable some debug features.

 */

#ifdef ARCH_DMA_MINALIGN

#define ARCH_KMALLOC_MINALIGN ARCH_DMA_MINALIGN

#else

#define ARCH_KMALLOC_MINALIGN __alignof__(unsigned long long)

#endif



#ifndef ARCH_SLAB_MINALIGN

/*

 * Enforce a minimum alignment for all caches.

 * Intended for archs that get misalignment faults even for BYTES_PER_WORD

 * aligned buffers. Includes ARCH_KMALLOC_MINALIGN.

 * If possible: Do not enable this flag for CONFIG_DEBUG_SLAB, it disables

 * some debug features.

 */

#define ARCH_SLAB_MINALIGN 0

#endif



/*

 * struct kmem_cache

 *
@@ -50,21 +24,19 @@ 
 */



struct kmem_cache {

/* 1) per-cpu data, touched during every alloc/free */

	struct array_cache *array[NR_CPUS];

/* 2) Cache tunables. Protected by cache_chain_mutex */

/* 1) Cache tunables. Protected by cache_chain_mutex */

	unsigned int batchcount;

	unsigned int limit;

	unsigned int shared;



	unsigned int buffer_size;

	u32 reciprocal_buffer_size;

/* 3) touched by every alloc & free from the backend */

/* 2) touched by every alloc & free from the backend */



	unsigned int flags;		/* constant flags */

	unsigned int num;		/* # of objs per slab */



/* 4) cache_grow/shrink */

/* 3) cache_grow/shrink */

	/* order of pgs per slab (2^n) */

	unsigned int gfporder;
@@ -80,11 +52,11 @@ struct kmem_cache { 
	/* constructor func */

	void (*ctor)(void *obj);



/* 5) cache creation/removal */

/* 4) cache creation/removal */

	const char *name;

	struct list_head next;



/* 6) statistics */

/* 5) statistics */

#ifdef CONFIG_DEBUG_SLAB

	unsigned long num_active;

	unsigned long num_allocations;
@@ -111,16 +83,18 @@ struct kmem_cache { 
	int obj_size;

#endif /* CONFIG_DEBUG_SLAB */



/* 6) per-cpu/per-node data, touched during every alloc/free */

	/*

	 * We put nodelists[] at the end of kmem_cache, because we want to size

	 * this array to nr_node_ids slots instead of MAX_NUMNODES

	 * We put array[] at the end of kmem_cache, because we want to size

	 * this array to nr_cpu_ids slots instead of NR_CPUS

	 * (see kmem_cache_init())

	 * We still use [MAX_NUMNODES] and not [1] or [0] because cache_cache

	 * is statically defined, so we reserve the max number of nodes.

	 * We still use [NR_CPUS] and not [1] or [0] because cache_cache

	 * is statically defined, so we reserve the max number of cpus.

	 */

	struct kmem_list3 *nodelists[MAX_NUMNODES];

	struct kmem_list3 **nodelists;

	struct array_cache *array[NR_CPUS];

	/*

	 * Do not add fields after nodelists[]

	 * Do not add fields after array[]

	 */

};

include/linux/slob_def.h

+0 −10
Original line number Diff line number Diff line 
#ifndef __LINUX_SLOB_DEF_H

#define __LINUX_SLOB_DEF_H



#ifdef ARCH_DMA_MINALIGN

#define ARCH_KMALLOC_MINALIGN ARCH_DMA_MINALIGN

#else

#define ARCH_KMALLOC_MINALIGN __alignof__(unsigned long)

#endif



#ifndef ARCH_SLAB_MINALIGN

#define ARCH_SLAB_MINALIGN __alignof__(unsigned long)

#endif



void *kmem_cache_alloc_node(struct kmem_cache *, gfp_t flags, int node);



static __always_inline void *kmem_cache_alloc(struct kmem_cache *cachep,

include/linux/slub_def.h

+13 −10
Original line number Diff line number Diff line
@@ -113,16 +113,6 @@ struct kmem_cache { 


#define KMALLOC_SHIFT_LOW ilog2(KMALLOC_MIN_SIZE)



#ifdef ARCH_DMA_MINALIGN

#define ARCH_KMALLOC_MINALIGN ARCH_DMA_MINALIGN

#else

#define ARCH_KMALLOC_MINALIGN __alignof__(unsigned long long)

#endif



#ifndef ARCH_SLAB_MINALIGN

#define ARCH_SLAB_MINALIGN __alignof__(unsigned long long)

#endif



/*

 * Maximum kmalloc object size handled by SLUB. Larger object allocations

 * are passed through to the page allocator. The page allocator "fastpath"
@@ -228,6 +218,19 @@ kmalloc_order(size_t size, gfp_t flags, unsigned int order) 
	return ret;

}



/**

 * Calling this on allocated memory will check that the memory

 * is expected to be in use, and print warnings if not.

 */

#ifdef CONFIG_SLUB_DEBUG

extern bool verify_mem_not_deleted(const void *x);

#else

static inline bool verify_mem_not_deleted(const void *x)

{

	return true;

}

#endif



#ifdef CONFIG_TRACING

extern void *

kmem_cache_alloc_trace(struct kmem_cache *s, gfp_t gfpflags, size_t size);

mm/slab.c

+9 −8
Original line number Diff line number Diff line
@@ -574,7 +574,9 @@ static struct arraycache_init initarray_generic = 
    { {0, BOOT_CPUCACHE_ENTRIES, 1, 0} };



/* internal cache of cache description objs */

static struct kmem_list3 *cache_cache_nodelists[MAX_NUMNODES];

static struct kmem_cache cache_cache = {

	.nodelists = cache_cache_nodelists,

	.batchcount = 1,

	.limit = BOOT_CPUCACHE_ENTRIES,

	.shared = 1,
@@ -1492,10 +1494,9 @@ void __init kmem_cache_init(void) 
	cache_cache.nodelists[node] = &initkmem_list3[CACHE_CACHE + node];



	/*

	 * struct kmem_cache size depends on nr_node_ids, which

	 * can be less than MAX_NUMNODES.

	 * struct kmem_cache size depends on nr_node_ids & nr_cpu_ids

	 */

	cache_cache.buffer_size = offsetof(struct kmem_cache, nodelists) +

	cache_cache.buffer_size = offsetof(struct kmem_cache, array[nr_cpu_ids]) +

				  nr_node_ids * sizeof(struct kmem_list3 *);

#if DEBUG

	cache_cache.obj_size = cache_cache.buffer_size;
@@ -2308,6 +2309,7 @@ kmem_cache_create (const char *name, size_t size, size_t align, 
	if (!cachep)

		goto oops;



	cachep->nodelists = (struct kmem_list3 **)&cachep->array[nr_cpu_ids];

#if DEBUG

	cachep->obj_size = size;
@@ -3153,12 +3155,11 @@ static void *cache_alloc_debugcheck_after(struct kmem_cache *cachep, 
	objp += obj_offset(cachep);

	if (cachep->ctor && cachep->flags & SLAB_POISON)

		cachep->ctor(objp);

#if ARCH_SLAB_MINALIGN

	if ((u32)objp & (ARCH_SLAB_MINALIGN-1)) {

	if (ARCH_SLAB_MINALIGN &&

	    ((unsigned long)objp & (ARCH_SLAB_MINALIGN-1))) {

		printk(KERN_ERR "0x%p: not aligned to ARCH_SLAB_MINALIGN=%d\n",

		       objp, ARCH_SLAB_MINALIGN);

		       objp, (int)ARCH_SLAB_MINALIGN);

	}

#endif

	return objp;

}

#else

CRA Git | Maintained and supported by SUSTech CRA and CCSE