SLUB: Use unique end pointer for each slab page.

#if NR_CPUS >= CONFIG_SPLIT_PTLOCK_CPUS

	    spinlock_t ptl;

#endif

	    struct {

		   struct kmem_cache *slab;	/* SLUB: Pointer to slab */

		   void *end;			/* SLUB: end marker */

	    };

	    struct page *first_page;	/* Compound tail pages */

	};

	union {

#endif

}

/*

 * The end pointer in a slab is special. It points to the first object in the

 * slab but has bit 0 set to mark it.

 *

 * Note that SLUB relies on page_mapping returning NULL for pages with bit 0

 * in the mapping set.

 */

static inline int is_end(void *addr)

{

	return (unsigned long)addr & PAGE_MAPPING_ANON;

}

void *slab_address(struct page *page)

{

	return page->end - PAGE_MAPPING_ANON;

}

static inline int check_valid_pointer(struct kmem_cache *s,

				struct page *page, const void *object)

{

	void *base;

	if (!object)

	if (object == page->end)

		return 1;

	base = page_address(page);

	base = slab_address(page);

	if (object < base || object >= base + s->objects * s->size ||

		(object - base) % s->size) {

		return 0;

/* Scan freelist */

#define for_each_free_object(__p, __s, __free) \

	for (__p = (__free); __p; __p = get_freepointer((__s), __p))

	for (__p = (__free); (__p) != page->end; __p = get_freepointer((__s),\

		__p))

/* Determine object index from a given position */

static inline int slab_index(void *p, struct kmem_cache *s, void *addr)

static void print_trailer(struct kmem_cache *s, struct page *page, u8 *p)

{

	unsigned int off;	/* Offset of last byte */

	u8 *addr = page_address(page);

	u8 *addr = slab_address(page);

	print_tracking(s, p);

	if (!(s->flags & SLAB_POISON))

		return 1;

	start = page_address(page);

	start = slab_address(page);

	end = start + (PAGE_SIZE << s->order);

	length = s->objects * s->size;

	remainder = end - (start + length);

		 * of the free objects in this slab. May cause

		 * another error because the object count is now wrong.

		 */

		set_freepointer(s, p, NULL);

		set_freepointer(s, p, page->end);

		return 0;

	}

	return 1;

	void *fp = page->freelist;

	void *object = NULL;

	while (fp && nr <= s->objects) {

	while (fp != page->end && nr <= s->objects) {

		if (fp == search)

			return 1;

		if (!check_valid_pointer(s, page, fp)) {

			if (object) {

				object_err(s, page, object,

					"Freechain corrupt");

				set_freepointer(s, object, NULL);

				set_freepointer(s, object, page->end);

				break;

			} else {

				slab_err(s, page, "Freepointer corrupt");

				page->freelist = NULL;

				page->freelist = page->end;

				page->inuse = s->objects;

				slab_fix(s, "Freelist cleared");

				return 0;

		 */

		slab_fix(s, "Marking all objects used");

		page->inuse = s->objects;

		page->freelist = NULL;

		page->freelist = page->end;

	}

	return 0;

}

	}

	/* Special debug activities for freeing objects */

	if (!SlabFrozen(page) && !page->freelist)

	if (!SlabFrozen(page) && page->freelist == page->end)

		remove_full(s, page);

	if (s->flags & SLAB_STORE_USER)

		set_track(s, object, TRACK_FREE, addr);

		SetSlabDebug(page);

	start = page_address(page);

	page->end = start + 1;

	if (unlikely(s->flags & SLAB_POISON))

		memset(start, POISON_INUSE, PAGE_SIZE << s->order);

		last = p;

	}

	setup_object(s, page, last);

	set_freepointer(s, last, NULL);

	set_freepointer(s, last, page->end);

	page->freelist = start;

	page->inuse = 0;

		void *p;

		slab_pad_check(s, page);

		for_each_object(p, s, page_address(page))

		for_each_object(p, s, slab_address(page))

			check_object(s, page, p, 0);

		ClearSlabDebug(page);

	}

		NR_SLAB_RECLAIMABLE : NR_SLAB_UNRECLAIMABLE,

		-pages);

	page->mapping = NULL;

	__free_pages(page, s->order);

}

	ClearSlabFrozen(page);

	if (page->inuse) {

		if (page->freelist)

		if (page->freelist != page->end)

			add_partial(n, page, tail);

		else if (SlabDebug(page) && (s->flags & SLAB_STORE_USER))

			add_full(n, page);

	 * Merge cpu freelist into freelist. Typically we get here

	 * because both freelists are empty. So this is unlikely

	 * to occur.

	 *

	 * We need to use _is_end here because deactivate slab may

	 * be called for a debug slab. Then c->freelist may contain

	 * a dummy pointer.

	 */

	while (unlikely(c->freelist)) {

	while (unlikely(!is_end(c->freelist))) {

		void **object;

		tail = 0;	/* Hot objects. Put the slab first */

		goto another_slab;

load_freelist:

	object = c->page->freelist;

	if (unlikely(!object))

	if (unlikely(object == c->page->end))

		goto another_slab;

	if (unlikely(SlabDebug(c->page)))

		goto debug;

	object = c->page->freelist;

	c->freelist = object[c->offset];

	c->page->inuse = s->objects;

	c->page->freelist = NULL;

	c->page->freelist = c->page->end;

	c->node = page_to_nid(c->page);

	slab_unlock(c->page);

	return object;

	local_irq_save(flags);

	c = get_cpu_slab(s, smp_processor_id());

	if (unlikely(!c->freelist || !node_match(c, node)))

	if (unlikely(is_end(c->freelist) || !node_match(c, node)))

		object = __slab_alloc(s, gfpflags, node, addr, c);

	 * was not on the partial list before

	 * then add it.

	 */

	if (unlikely(!prior))

	if (unlikely(prior == page->end))

		add_partial(get_node(s, page_to_nid(page)), page, 1);

out_unlock:

	return;

slab_empty:

	if (prior)

	if (prior != page->end)

		/*

		 * Slab still on the partial list.

		 */

			struct kmem_cache_cpu *c)

{

	c->page = NULL;

	c->freelist = NULL;

	c->freelist = (void *)PAGE_MAPPING_ANON;

	c->node = 0;

	c->offset = s->offset / sizeof(void *);

	c->objsize = s->objsize;

						unsigned long *map)

{

	void *p;

	void *addr = page_address(page);

	void *addr = slab_address(page);

	if (!check_slab(s, page) ||

			!on_freelist(s, page, NULL))

static void process_slab(struct loc_track *t, struct kmem_cache *s,

		struct page *page, enum track_item alloc)

{

	void *addr = page_address(page);

	void *addr = slab_address(page);

	DECLARE_BITMAP(map, s->objects);

	void *p;