Merge branch 'slab/for-linus' of git://git.kernel.org/pub/scm/linux/kernel/git/penberg/linux (138c4ae9) · Commits · 戴 / test

Documentation/vm/00-INDEX

+0 −2

Original line number	Diff line number	Diff line
		@@ -30,8 +30,6 @@ page_migration
		- description of page migration in NUMA systems.
		pagemap.txt
		- pagemap, from the userspace perspective
		slabinfo.c
		- source code for a tool to get reports about slabs.
		slub.txt
		- a short users guide for SLUB.
		unevictable-lru.txt

include/linux/mm_types.h

+13 −1

Original line number	Diff line number	Diff line
		@@ -79,9 +79,21 @@ struct page {
		};

		/* Third double word block */
		union {
		struct list_head lru; /* Pageout list, eg. active_list
		* protected by zone->lru_lock !
		*/
		struct { /* slub per cpu partial pages */
		struct page next; / Next partial slab */
		#ifdef CONFIG_64BIT
		int pages; /* Nr of partial slabs left */
		int pobjects; /* Approximate # of objects */
		#else
		short int pages;
		short int pobjects;
		#endif
		};
		};

		/* Remainder is not double word aligned */
		union {

include/linux/slub_def.h

+4 −0

Original line number	Diff line number	Diff line
		@@ -36,12 +36,15 @@ enum stat_item {
		ORDER_FALLBACK, /* Number of times fallback was necessary */
		CMPXCHG_DOUBLE_CPU_FAIL,/* Failure of this_cpu_cmpxchg_double */
		CMPXCHG_DOUBLE_FAIL, /* Number of times that cmpxchg double did not match */
		CPU_PARTIAL_ALLOC, /* Used cpu partial on alloc */
		CPU_PARTIAL_FREE, /* USed cpu partial on free */
		NR_SLUB_STAT_ITEMS };

		struct kmem_cache_cpu {
		void *freelist; / Pointer to next available object */
		unsigned long tid; /* Globally unique transaction id */
		struct page page; / The slab from which we are allocating */
		struct page partial; / Partially allocated frozen slabs */
		int node; /* The node of the page (or -1 for debug) */
		#ifdef CONFIG_SLUB_STATS
		unsigned stat[NR_SLUB_STAT_ITEMS];
		@@ -79,6 +82,7 @@ struct kmem_cache {
		int size; /* The size of an object including meta data */
		int objsize; /* The size of an object without meta data */
		int offset; /* Free pointer offset. */
		int cpu_partial; /* Number of per cpu partial objects to keep around */
		struct kmem_cache_order_objects oo;

		/* Allocation and freeing of slabs */

mm/slab.c

+7 −12

Original line number	Diff line number	Diff line
		@@ -1857,9 +1857,9 @@ static void dump_line(char *data, int offset, int limit)
		error = data[offset + i];
		bad_count++;
		}
		printk(" %02x", (unsigned char)data[offset + i]);
		}
		printk("\n");
		print_hex_dump(KERN_CONT, "", 0, 16, 1,
		&data[offset], limit, 1);

		if (bad_count == 1) {
		error ^= POISON_FREE;
		@@ -3039,14 +3039,9 @@ bad:
		printk(KERN_ERR "slab: Internal list corruption detected in "
		"cache '%s'(%d), slabp %p(%d). Hexdump:\n",
		cachep->name, cachep->num, slabp, slabp->inuse);
		for (i = 0;
		i < sizeof(slabp) + cachep->num sizeof(kmem_bufctl_t);
		i++) {
		if (i % 16 == 0)
		printk("\n%03x:", i);
		printk(" %02x", ((unsigned char *)slabp)[i]);
		}
		printk("\n");
		print_hex_dump(KERN_ERR, "", DUMP_PREFIX_OFFSET, 16, 1, slabp,
		sizeof(slabp) + cachep->num sizeof(kmem_bufctl_t),
		1);
		BUG();
		}
		}
		@@ -4584,7 +4579,7 @@ static const struct file_operations proc_slabstats_operations = {

		static int __init slab_proc_init(void)
		{
		proc_create("slabinfo",S_IWUSR\|S_IRUGO,NULL,&proc_slabinfo_operations);
		proc_create("slabinfo",S_IWUSR\|S_IRUSR,NULL,&proc_slabinfo_operations);
		#ifdef CONFIG_DEBUG_SLAB_LEAK
		proc_create("slab_allocators", 0, NULL, &proc_slabstats_operations);
		#endif

mm/slub.c

+392 −166

Original line number	Diff line number	Diff line
		@@ -467,34 +467,8 @@ static int disable_higher_order_debug;
		*/
		static void print_section(char text, u8 addr, unsigned int length)
		{
		int i, offset;
		int newline = 1;
		char ascii[17];

		ascii[16] = 0;

		for (i = 0; i < length; i++) {
		if (newline) {
		printk(KERN_ERR "%8s 0x%p: ", text, addr + i);
		newline = 0;
		}
		printk(KERN_CONT " %02x", addr[i]);
		offset = i % 16;
		ascii[offset] = isgraph(addr[i]) ? addr[i] : '.';
		if (offset == 15) {
		printk(KERN_CONT " %s\n", ascii);
		newline = 1;
		}
		}
		if (!newline) {
		i %= 16;
		while (i < 16) {
		printk(KERN_CONT " ");
		ascii[i] = ' ';
		i++;
		}
		printk(KERN_CONT " %s\n", ascii);
		}
		print_hex_dump(KERN_ERR, text, DUMP_PREFIX_ADDRESS, 16, 1, addr,
		length, 1);
		}

		static struct track get_track(struct kmem_cache s, void *object,
		@@ -627,8 +601,8 @@ static void print_trailer(struct kmem_cache s, struct page page, u8 *p)
		if (p > addr + 16)
		print_section("Bytes b4 ", p - 16, 16);

		print_section("Object", p, min_t(unsigned long, s->objsize, PAGE_SIZE));

		print_section("Object ", p, min_t(unsigned long, s->objsize,
		PAGE_SIZE));
		if (s->flags & SLAB_RED_ZONE)
		print_section("Redzone ", p + s->objsize,
		s->inuse - s->objsize);
		@@ -1447,7 +1421,7 @@ static struct page new_slab(struct kmem_cache s, gfp_t flags, int node)
		set_freepointer(s, last, NULL);

		page->freelist = start;
		page->inuse = 0;
		page->inuse = page->objects;
		page->frozen = 1;
		out:
		return page;
		@@ -1534,7 +1508,7 @@ static inline void add_partial(struct kmem_cache_node *n,
		struct page *page, int tail)
		{
		n->nr_partial++;
		if (tail)
		if (tail == DEACTIVATE_TO_TAIL)
		list_add_tail(&page->lru, &n->partial);
		else
		list_add(&page->lru, &n->partial);
		@@ -1554,10 +1528,13 @@ static inline void remove_partial(struct kmem_cache_node *n,
		* Lock slab, remove from the partial list and put the object into the
		* per cpu freelist.
		*
		* Returns a list of objects or NULL if it fails.
		*
		* Must hold list_lock.
		*/
		static inline int acquire_slab(struct kmem_cache *s,
		struct kmem_cache_node n, struct page page)
		static inline void acquire_slab(struct kmem_cache s,
		struct kmem_cache_node n, struct page page,
		int mode)
		{
		void *freelist;
		unsigned long counters;
		@@ -1572,6 +1549,7 @@ static inline int acquire_slab(struct kmem_cache *s,
		freelist = page->freelist;
		counters = page->counters;
		new.counters = counters;
		if (mode)
		new.inuse = page->objects;

		VM_BUG_ON(new.frozen);
		@@ -1583,32 +1561,19 @@ static inline int acquire_slab(struct kmem_cache *s,
		"lock and freeze"));

		remove_partial(n, page);

		if (freelist) {
		/* Populate the per cpu freelist */
		this_cpu_write(s->cpu_slab->freelist, freelist);
		this_cpu_write(s->cpu_slab->page, page);
		this_cpu_write(s->cpu_slab->node, page_to_nid(page));
		return 1;
		} else {
		/*
		* Slab page came from the wrong list. No object to allocate
		* from. Put it onto the correct list and continue partial
		* scan.
		*/
		printk(KERN_ERR "SLUB: %s : Page without available objects on"
		" partial list\n", s->name);
		return 0;
		}
		return freelist;
		}

		static int put_cpu_partial(struct kmem_cache s, struct page page, int drain);

		/*
		* Try to allocate a partial slab from a specific node.
		*/
		static struct page get_partial_node(struct kmem_cache s,
		struct kmem_cache_node *n)
		static void get_partial_node(struct kmem_cache s,
		struct kmem_cache_node n, struct kmem_cache_cpu c)
		{
		struct page *page;
		struct page page, page2;
		void *object = NULL;

		/*
		* Racy check. If we mistakenly see no partial slabs then we
		@@ -1620,26 +1585,43 @@ static struct page get_partial_node(struct kmem_cache s,
		return NULL;

		spin_lock(&n->list_lock);
		list_for_each_entry(page, &n->partial, lru)
		if (acquire_slab(s, n, page))
		goto out;
		page = NULL;
		out:
		list_for_each_entry_safe(page, page2, &n->partial, lru) {
		void *t = acquire_slab(s, n, page, object == NULL);
		int available;

		if (!t)
		break;

		if (!object) {
		c->page = page;
		c->node = page_to_nid(page);
		stat(s, ALLOC_FROM_PARTIAL);
		object = t;
		available = page->objects - page->inuse;
		} else {
		page->freelist = t;
		available = put_cpu_partial(s, page, 0);
		}
		if (kmem_cache_debug(s) \|\| available > s->cpu_partial / 2)
		break;

		}
		spin_unlock(&n->list_lock);
		return page;
		return object;
		}

		/*
		* Get a page from somewhere. Search in increasing NUMA distances.
		*/
		static struct page get_any_partial(struct kmem_cache s, gfp_t flags)
		static struct page get_any_partial(struct kmem_cache s, gfp_t flags,
		struct kmem_cache_cpu *c)
		{
		#ifdef CONFIG_NUMA
		struct zonelist *zonelist;
		struct zoneref *z;
		struct zone *zone;
		enum zone_type high_zoneidx = gfp_zone(flags);
		struct page *page;
		void *object;

		/*
		* The defrag ratio allows a configuration of the tradeoffs between
		@@ -1672,10 +1654,10 @@ static struct page get_any_partial(struct kmem_cache s, gfp_t flags)

		if (n && cpuset_zone_allowed_hardwall(zone, flags) &&
		n->nr_partial > s->min_partial) {
		page = get_partial_node(s, n);
		if (page) {
		object = get_partial_node(s, n, c);
		if (object) {
		put_mems_allowed();
		return page;
		return object;
		}
		}
		}
		@@ -1687,16 +1669,17 @@ static struct page get_any_partial(struct kmem_cache s, gfp_t flags)
		/*
		* Get a partial page, lock it and return it.
		*/
		static struct page get_partial(struct kmem_cache s, gfp_t flags, int node)
		static void get_partial(struct kmem_cache s, gfp_t flags, int node,
		struct kmem_cache_cpu *c)
		{
		struct page *page;
		void *object;
		int searchnode = (node == NUMA_NO_NODE) ? numa_node_id() : node;

		page = get_partial_node(s, get_node(s, searchnode));
		if (page \|\| node != NUMA_NO_NODE)
		return page;
		object = get_partial_node(s, get_node(s, searchnode), c);
		if (object \|\| node != NUMA_NO_NODE)
		return object;

		return get_any_partial(s, flags);
		return get_any_partial(s, flags, c);
		}

		#ifdef CONFIG_PREEMPT
		@@ -1765,9 +1748,6 @@ void init_kmem_cache_cpus(struct kmem_cache *s)
		for_each_possible_cpu(cpu)
		per_cpu_ptr(s->cpu_slab, cpu)->tid = init_tid(cpu);
		}
		/*
		* Remove the cpu slab
		*/

		/*
		* Remove the cpu slab
		@@ -1781,13 +1761,13 @@ static void deactivate_slab(struct kmem_cache s, struct kmem_cache_cpu c)
		enum slab_modes l = M_NONE, m = M_NONE;
		void *freelist;
		void *nextfree;
		int tail = 0;
		int tail = DEACTIVATE_TO_HEAD;
		struct page new;
		struct page old;

		if (page->freelist) {
		stat(s, DEACTIVATE_REMOTE_FREES);
		tail = 1;
		tail = DEACTIVATE_TO_TAIL;
		}

		c->tid = next_tid(c->tid);
		@@ -1893,7 +1873,7 @@ redo:
		if (m == M_PARTIAL) {

		add_partial(n, page, tail);
		stat(s, tail ? DEACTIVATE_TO_TAIL : DEACTIVATE_TO_HEAD);
		stat(s, tail);

		} else if (m == M_FULL) {

		@@ -1920,6 +1900,123 @@ redo:
		}
		}

		/* Unfreeze all the cpu partial slabs */
		static void unfreeze_partials(struct kmem_cache *s)
		{
		struct kmem_cache_node *n = NULL;
		struct kmem_cache_cpu *c = this_cpu_ptr(s->cpu_slab);
		struct page *page;

		while ((page = c->partial)) {
		enum slab_modes { M_PARTIAL, M_FREE };
		enum slab_modes l, m;
		struct page new;
		struct page old;

		c->partial = page->next;
		l = M_FREE;

		do {

		old.freelist = page->freelist;
		old.counters = page->counters;
		VM_BUG_ON(!old.frozen);

		new.counters = old.counters;
		new.freelist = old.freelist;

		new.frozen = 0;

		if (!new.inuse && (!n \|\| n->nr_partial > s->min_partial))
		m = M_FREE;
		else {
		struct kmem_cache_node *n2 = get_node(s,
		page_to_nid(page));

		m = M_PARTIAL;
		if (n != n2) {
		if (n)
		spin_unlock(&n->list_lock);

		n = n2;
		spin_lock(&n->list_lock);
		}
		}

		if (l != m) {
		if (l == M_PARTIAL)
		remove_partial(n, page);
		else
		add_partial(n, page, 1);

		l = m;
		}

		} while (!cmpxchg_double_slab(s, page,
		old.freelist, old.counters,
		new.freelist, new.counters,
		"unfreezing slab"));

		if (m == M_FREE) {
		stat(s, DEACTIVATE_EMPTY);
		discard_slab(s, page);
		stat(s, FREE_SLAB);
		}
		}

		if (n)
		spin_unlock(&n->list_lock);
		}

		/*
		* Put a page that was just frozen (in __slab_free) into a partial page
		* slot if available. This is done without interrupts disabled and without
		* preemption disabled. The cmpxchg is racy and may put the partial page
		* onto a random cpus partial slot.
		*
		* If we did not find a slot then simply move all the partials to the
		* per node partial list.
		*/
		int put_cpu_partial(struct kmem_cache s, struct page page, int drain)
		{
		struct page *oldpage;
		int pages;
		int pobjects;

		do {
		pages = 0;
		pobjects = 0;
		oldpage = this_cpu_read(s->cpu_slab->partial);

		if (oldpage) {
		pobjects = oldpage->pobjects;
		pages = oldpage->pages;
		if (drain && pobjects > s->cpu_partial) {
		unsigned long flags;
		/*
		* partial array is full. Move the existing
		* set to the per node partial list.
		*/
		local_irq_save(flags);
		unfreeze_partials(s);
		local_irq_restore(flags);
		pobjects = 0;
		pages = 0;
		}
		}

		pages++;
		pobjects += page->objects - page->inuse;

		page->pages = pages;
		page->pobjects = pobjects;
		page->next = oldpage;

		} while (this_cpu_cmpxchg(s->cpu_slab->partial, oldpage, page) != oldpage);
		stat(s, CPU_PARTIAL_FREE);
		return pobjects;
		}

		static inline void flush_slab(struct kmem_cache s, struct kmem_cache_cpu c)
		{
		stat(s, CPUSLAB_FLUSH);
		@@ -1935,8 +2032,12 @@ static inline void __flush_cpu_slab(struct kmem_cache *s, int cpu)
		{
		struct kmem_cache_cpu *c = per_cpu_ptr(s->cpu_slab, cpu);

		if (likely(c && c->page))
		if (likely(c)) {
		if (c->page)
		flush_slab(s, c);

		unfreeze_partials(s);
		}
		}

		static void flush_cpu_slab(void *d)
		@@ -2027,12 +2128,39 @@ slab_out_of_memory(struct kmem_cache *s, gfp_t gfpflags, int nid)
		}
		}

		static inline void new_slab_objects(struct kmem_cache s, gfp_t flags,
		int node, struct kmem_cache_cpu **pc)
		{
		void *object;
		struct kmem_cache_cpu *c;
		struct page *page = new_slab(s, flags, node);

		if (page) {
		c = __this_cpu_ptr(s->cpu_slab);
		if (c->page)
		flush_slab(s, c);

		/*
		* No other reference to the page yet so we can
		* muck around with it freely without cmpxchg
		*/
		object = page->freelist;
		page->freelist = NULL;

		stat(s, ALLOC_SLAB);
		c->node = page_to_nid(page);
		c->page = page;
		*pc = c;
		} else
		object = NULL;

		return object;
		}

		/*
		* Slow path. The lockless freelist is empty or we need to perform
		* debugging duties.
		*
		* Interrupts are disabled.
		*
		* Processing is still very fast if new objects have been freed to the
		* regular freelist. In that case we simply take over the regular freelist
		* as the lockless freelist and zap the regular freelist.
		@@ -2049,7 +2177,6 @@ static void __slab_alloc(struct kmem_cache s, gfp_t gfpflags, int node,
		unsigned long addr, struct kmem_cache_cpu *c)
		{
		void **object;
		struct page *page;
		unsigned long flags;
		struct page new;
		unsigned long counters;
		@@ -2064,13 +2191,9 @@ static void __slab_alloc(struct kmem_cache s, gfp_t gfpflags, int node,
		c = this_cpu_ptr(s->cpu_slab);
		#endif

		/* We handle __GFP_ZERO in the caller */
		gfpflags &= ~__GFP_ZERO;

		page = c->page;
		if (!page)
		if (!c->page)
		goto new_slab;

		redo:
		if (unlikely(!node_match(c, node))) {
		stat(s, ALLOC_NODE_MISMATCH);
		deactivate_slab(s, c);
		@@ -2080,8 +2203,8 @@ static void __slab_alloc(struct kmem_cache s, gfp_t gfpflags, int node,
		stat(s, ALLOC_SLOWPATH);

		do {
		object = page->freelist;
		counters = page->counters;
		object = c->page->freelist;
		counters = c->page->counters;
		new.counters = counters;
		VM_BUG_ON(!new.frozen);

		@@ -2095,15 +2218,15 @@ static void __slab_alloc(struct kmem_cache s, gfp_t gfpflags, int node,
		* and use them to refill the per cpu queue.
		*/

		new.inuse = page->objects;
		new.inuse = c->page->objects;
		new.frozen = object != NULL;

		} while (!__cmpxchg_double_slab(s, page,
		} while (!__cmpxchg_double_slab(s, c->page,
		object, counters,
		NULL, new.counters,
		"__slab_alloc"));

		if (unlikely(!object)) {
		if (!object) {
		c->page = NULL;
		stat(s, DEACTIVATE_BYPASS);
		goto new_slab;
		@@ -2112,58 +2235,47 @@ static void __slab_alloc(struct kmem_cache s, gfp_t gfpflags, int node,
		stat(s, ALLOC_REFILL);

		load_freelist:
		VM_BUG_ON(!page->frozen);
		c->freelist = get_freepointer(s, object);
		c->tid = next_tid(c->tid);
		local_irq_restore(flags);
		return object;

		new_slab:
		page = get_partial(s, gfpflags, node);
		if (page) {
		stat(s, ALLOC_FROM_PARTIAL);
		object = c->freelist;

		if (kmem_cache_debug(s))
		goto debug;
		goto load_freelist;
		if (c->partial) {
		c->page = c->partial;
		c->partial = c->page->next;
		c->node = page_to_nid(c->page);
		stat(s, CPU_PARTIAL_ALLOC);
		c->freelist = NULL;
		goto redo;
		}

		page = new_slab(s, gfpflags, node);
		/* Then do expensive stuff like retrieving pages from the partial lists */
		object = get_partial(s, gfpflags, node, c);

		if (page) {
		c = __this_cpu_ptr(s->cpu_slab);
		if (c->page)
		flush_slab(s, c);

		/*
		* No other reference to the page yet so we can
		* muck around with it freely without cmpxchg
		*/
		object = page->freelist;
		page->freelist = NULL;
		page->inuse = page->objects;
		if (unlikely(!object)) {

		stat(s, ALLOC_SLAB);
		c->node = page_to_nid(page);
		c->page = page;
		object = new_slab_objects(s, gfpflags, node, &c);

		if (kmem_cache_debug(s))
		goto debug;
		goto load_freelist;
		}
		if (unlikely(!object)) {
		if (!(gfpflags & __GFP_NOWARN) && printk_ratelimit())
		slab_out_of_memory(s, gfpflags, node);

		local_irq_restore(flags);
		return NULL;
		}
		}

		debug:
		if (!object \|\| !alloc_debug_processing(s, page, object, addr))
		goto new_slab;
		if (likely(!kmem_cache_debug(s)))
		goto load_freelist;

		/* Only entered in the debug case */
		if (!alloc_debug_processing(s, c->page, object, addr))
		goto new_slab; /* Slab failed checks. Next slab needed */

		c->freelist = get_freepointer(s, object);
		deactivate_slab(s, c);
		c->page = NULL;
		c->node = NUMA_NO_NODE;
		local_irq_restore(flags);
		return object;
		@@ -2333,6 +2445,17 @@ static void __slab_free(struct kmem_cache s, struct page page,
		was_frozen = new.frozen;
		new.inuse--;
		if ((!new.inuse \|\| !prior) && !was_frozen && !n) {

		if (!kmem_cache_debug(s) && !prior)

		/*
		* Slab was on no list before and will be partially empty
		* We can defer the list move and instead freeze it.
		*/
		new.frozen = 1;

		else { /* Needs to be taken off a list */

		n = get_node(s, page_to_nid(page));
		/*
		* Speculatively acquire the list_lock.
		@@ -2343,6 +2466,8 @@ static void __slab_free(struct kmem_cache s, struct page page,
		* other processors updating the list of slabs.
		*/
		spin_lock_irqsave(&n->list_lock, flags);

		}
		}
		inuse = new.inuse;

		@@ -2352,6 +2477,14 @@ static void __slab_free(struct kmem_cache s, struct page page,
		"__slab_free"));

		if (likely(!n)) {

		/*
		* If we just froze the page then put it onto the
		* per cpu partial list.
		*/
		if (new.frozen && !was_frozen)
		put_cpu_partial(s, page, 1);

		/*
		* The list lock was not taken therefore no list
		* activity can be necessary.
		@@ -2377,7 +2510,7 @@ static void __slab_free(struct kmem_cache s, struct page page,
		*/
		if (unlikely(!prior)) {
		remove_full(s, page);
		add_partial(n, page, 1);
		add_partial(n, page, DEACTIVATE_TO_TAIL);
		stat(s, FREE_ADD_PARTIAL);
		}
		}
		@@ -2421,7 +2554,6 @@ static __always_inline void slab_free(struct kmem_cache *s,
		slab_free_hook(s, x);

		redo:

		/*
		* Determine the currently cpus per cpu slab.
		* The cpu may change afterward. However that does not matter since
		@@ -2685,7 +2817,7 @@ static void early_kmem_cache_node_alloc(int node)
		n = page->freelist;
		BUG_ON(!n);
		page->freelist = get_freepointer(kmem_cache_node, n);
		page->inuse++;
		page->inuse = 1;
		page->frozen = 0;
		kmem_cache_node->node[node] = n;
		#ifdef CONFIG_SLUB_DEBUG
		@@ -2695,7 +2827,7 @@ static void early_kmem_cache_node_alloc(int node)
		init_kmem_cache_node(n, kmem_cache_node);
		inc_slabs_node(kmem_cache_node, node, page->objects);

		add_partial(n, page, 0);
		add_partial(n, page, DEACTIVATE_TO_HEAD);
		}

		static void free_kmem_cache_nodes(struct kmem_cache *s)
		@@ -2911,7 +3043,34 @@ static int kmem_cache_open(struct kmem_cache *s,
		* The larger the object size is, the more pages we want on the partial
		* list to avoid pounding the page allocator excessively.
		*/
		set_min_partial(s, ilog2(s->size));
		set_min_partial(s, ilog2(s->size) / 2);

		/*
		* cpu_partial determined the maximum number of objects kept in the
		* per cpu partial lists of a processor.
		*
		* Per cpu partial lists mainly contain slabs that just have one
		* object freed. If they are used for allocation then they can be
		* filled up again with minimal effort. The slab will never hit the
		* per node partial lists and therefore no locking will be required.
		*
		* This setting also determines
		*
		* A) The number of objects from per cpu partial slabs dumped to the
		* per node list when we reach the limit.
		* B) The number of objects in cpu partial slabs to extract from the
		* per node list when we run out of per cpu objects. We only fetch 50%
		* to keep some capacity around for frees.
		*/
		if (s->size >= PAGE_SIZE)
		s->cpu_partial = 2;
		else if (s->size >= 1024)
		s->cpu_partial = 6;
		else if (s->size >= 256)
		s->cpu_partial = 13;
		else
		s->cpu_partial = 30;

		s->refcount = 1;
		#ifdef CONFIG_NUMA
		s->remote_node_defrag_ratio = 1000;
		@@ -2970,13 +3129,13 @@ static void list_slab_objects(struct kmem_cache s, struct page page,

		/*
		* Attempt to free all partial slabs on a node.
		* This is called from kmem_cache_close(). We must be the last thread
		* using the cache and therefore we do not need to lock anymore.
		*/
		static void free_partial(struct kmem_cache s, struct kmem_cache_node n)
		{
		unsigned long flags;
		struct page page, h;

		spin_lock_irqsave(&n->list_lock, flags);
		list_for_each_entry_safe(page, h, &n->partial, lru) {
		if (!page->inuse) {
		remove_partial(n, page);
		@@ -2986,7 +3145,6 @@ static void free_partial(struct kmem_cache s, struct kmem_cache_node n)
		"Objects remaining on kmem_cache_close()");
		}
		}
		spin_unlock_irqrestore(&n->list_lock, flags);
		}

		/*
		@@ -3020,6 +3178,7 @@ void kmem_cache_destroy(struct kmem_cache *s)
		s->refcount--;
		if (!s->refcount) {
		list_del(&s->list);
		up_write(&slub_lock);
		if (kmem_cache_close(s)) {
		printk(KERN_ERR "SLUB %s: %s called for cache that "
		"still has objects.\n", s->name, __func__);
		@@ -3028,7 +3187,7 @@ void kmem_cache_destroy(struct kmem_cache *s)
		if (s->flags & SLAB_DESTROY_BY_RCU)
		rcu_barrier();
		sysfs_slab_remove(s);
		}
		} else
		up_write(&slub_lock);
		}
		EXPORT_SYMBOL(kmem_cache_destroy);
		@@ -3347,23 +3506,23 @@ int kmem_cache_shrink(struct kmem_cache *s)
		* list_lock. page->inuse here is the upper limit.
		*/
		list_for_each_entry_safe(page, t, &n->partial, lru) {
		if (!page->inuse) {
		remove_partial(n, page);
		discard_slab(s, page);
		} else {
		list_move(&page->lru,
		slabs_by_inuse + page->inuse);
		}
		list_move(&page->lru, slabs_by_inuse + page->inuse);
		if (!page->inuse)
		n->nr_partial--;
		}

		/*
		* Rebuild the partial list with the slabs filled up most
		* first and the least used slabs at the end.
		*/
		for (i = objects - 1; i >= 0; i--)
		for (i = objects - 1; i > 0; i--)
		list_splice(slabs_by_inuse + i, n->partial.prev);

		spin_unlock_irqrestore(&n->list_lock, flags);

		/* Release empty slabs */
		list_for_each_entry_safe(page, t, slabs_by_inuse, lru)
		discard_slab(s, page);
		}

		kfree(slabs_by_inuse);
		@@ -4319,6 +4478,7 @@ static ssize_t show_slab_objects(struct kmem_cache *s,

		for_each_possible_cpu(cpu) {
		struct kmem_cache_cpu *c = per_cpu_ptr(s->cpu_slab, cpu);
		struct page *page;

		if (!c \|\| c->node < 0)
		continue;
		@@ -4334,6 +4494,13 @@ static ssize_t show_slab_objects(struct kmem_cache *s,
		total += x;
		nodes[c->node] += x;
		}
		page = c->partial;

		if (page) {
		x = page->pobjects;
		total += x;
		nodes[c->node] += x;
		}
		per_cpu[c->node]++;
		}
		}
		@@ -4412,11 +4579,12 @@ struct slab_attribute {
		};

		#define SLAB_ATTR_RO(_name) \
		static struct slab_attribute _name##_attr = __ATTR_RO(_name)
		static struct slab_attribute _name##_attr = \
		__ATTR(_name, 0400, _name##_show, NULL)

		#define SLAB_ATTR(_name) \
		static struct slab_attribute _name##_attr = \
		__ATTR(_name, 0644, _name##_show, _name##_store)
		__ATTR(_name, 0600, _name##_show, _name##_store)

		static ssize_t slab_size_show(struct kmem_cache s, char buf)
		{
		@@ -4485,6 +4653,27 @@ static ssize_t min_partial_store(struct kmem_cache s, const char buf,
		}
		SLAB_ATTR(min_partial);

		static ssize_t cpu_partial_show(struct kmem_cache s, char buf)
		{
		return sprintf(buf, "%u\n", s->cpu_partial);
		}

		static ssize_t cpu_partial_store(struct kmem_cache s, const char buf,
		size_t length)
		{
		unsigned long objects;
		int err;

		err = strict_strtoul(buf, 10, &objects);
		if (err)
		return err;

		s->cpu_partial = objects;
		flush_all(s);
		return length;
		}
		SLAB_ATTR(cpu_partial);

		static ssize_t ctor_show(struct kmem_cache s, char buf)
		{
		if (!s->ctor)
		@@ -4523,6 +4712,37 @@ static ssize_t objects_partial_show(struct kmem_cache s, char buf)
		}
		SLAB_ATTR_RO(objects_partial);

		static ssize_t slabs_cpu_partial_show(struct kmem_cache s, char buf)
		{
		int objects = 0;
		int pages = 0;
		int cpu;
		int len;

		for_each_online_cpu(cpu) {
		struct page *page = per_cpu_ptr(s->cpu_slab, cpu)->partial;

		if (page) {
		pages += page->pages;
		objects += page->pobjects;
		}
		}

		len = sprintf(buf, "%d(%d)", objects, pages);

		#ifdef CONFIG_SMP
		for_each_online_cpu(cpu) {
		struct page *page = per_cpu_ptr(s->cpu_slab, cpu) ->partial;

		if (page && len < PAGE_SIZE - 20)
		len += sprintf(buf + len, " C%d=%d(%d)", cpu,
		page->pobjects, page->pages);
		}
		#endif
		return len + sprintf(buf + len, "\n");
		}
		SLAB_ATTR_RO(slabs_cpu_partial);

		static ssize_t reclaim_account_show(struct kmem_cache s, char buf)
		{
		return sprintf(buf, "%d\n", !!(s->flags & SLAB_RECLAIM_ACCOUNT));
		@@ -4845,6 +5065,8 @@ STAT_ATTR(DEACTIVATE_BYPASS, deactivate_bypass);
		STAT_ATTR(ORDER_FALLBACK, order_fallback);
		STAT_ATTR(CMPXCHG_DOUBLE_CPU_FAIL, cmpxchg_double_cpu_fail);
		STAT_ATTR(CMPXCHG_DOUBLE_FAIL, cmpxchg_double_fail);
		STAT_ATTR(CPU_PARTIAL_ALLOC, cpu_partial_alloc);
		STAT_ATTR(CPU_PARTIAL_FREE, cpu_partial_free);
		#endif

		static struct attribute *slab_attrs[] = {
		@@ -4853,6 +5075,7 @@ static struct attribute *slab_attrs[] = {
		&objs_per_slab_attr.attr,
		&order_attr.attr,
		&min_partial_attr.attr,
		&cpu_partial_attr.attr,
		&objects_attr.attr,
		&objects_partial_attr.attr,
		&partial_attr.attr,
		@@ -4865,6 +5088,7 @@ static struct attribute *slab_attrs[] = {
		&destroy_by_rcu_attr.attr,
		&shrink_attr.attr,
		&reserved_attr.attr,
		&slabs_cpu_partial_attr.attr,
		#ifdef CONFIG_SLUB_DEBUG
		&total_objects_attr.attr,
		&slabs_attr.attr,
		@@ -4906,6 +5130,8 @@ static struct attribute *slab_attrs[] = {
		&order_fallback_attr.attr,
		&cmpxchg_double_fail_attr.attr,
		&cmpxchg_double_cpu_fail_attr.attr,
		&cpu_partial_alloc_attr.attr,
		&cpu_partial_free_attr.attr,
		#endif
		#ifdef CONFIG_FAILSLAB
		&failslab_attr.attr,
		@@ -5257,7 +5483,7 @@ static const struct file_operations proc_slabinfo_operations = {

		static int __init slab_proc_init(void)
		{
		proc_create("slabinfo", S_IRUGO, NULL, &proc_slabinfo_operations);
		proc_create("slabinfo", S_IRUSR, NULL, &proc_slabinfo_operations);
		return 0;
		}
		module_init(slab_proc_init);

Admin message