Commit b35f1819 authored by Kirill A. Shutemov's avatar Kirill A. Shutemov Committed by Linus Torvalds
Browse files

mm: create a separate slab for page->ptl allocation 



If DEBUG_SPINLOCK and DEBUG_LOCK_ALLOC are enabled spinlock_t on x86_64
is 72 bytes.  For page->ptl they will be allocated from kmalloc-96 slab,
so we loose 24 on each.  An average system can easily allocate few tens
thousands of page->ptl and overhead is significant.

Let's create a separate slab for page->ptl allocation to solve this.

To make sure that it really works this time, some numbers from my test
machine (just booted, no load):

Before:
  # grep '^\(kmalloc-96\|page->ptl\)' /proc/slabinfo
  kmalloc-96         31987  32190    128   30    1 : tunables  120   60    8 : slabdata   1073   1073     92
After:
  # grep '^\(kmalloc-96\|page->ptl\)' /proc/slabinfo
  page->ptl          27516  28143     72   53    1 : tunables  120   60    8 : slabdata    531    531      9
  kmalloc-96          3853   5280    128   30    1 : tunables  120   60    8 : slabdata    176    176      0

Note that the patch is useful not only for debug case, but also for
PREEMPT_RT, where spinlock_t is always bloated.

Signed-off-by: default avatarKirill A. Shutemov <kirill.shutemov@linux.intel.com>
Signed-off-by: default avatarAndrew Morton <akpm@linux-foundation.org>
Signed-off-by: default avatarLinus Torvalds <torvalds@linux-foundation.org>
parent 943dca1a

include/linux/mm.h

+12 −0
Original line number Original line Diff line number Diff line
@@ -1350,6 +1350,7 @@ static inline pmd_t *pmd_alloc(struct mm_struct *mm, pud_t *pud, unsigned long a 




#if USE_SPLIT_PTE_PTLOCKS

#if USE_SPLIT_PTE_PTLOCKS

#if ALLOC_SPLIT_PTLOCKS

#if ALLOC_SPLIT_PTLOCKS

void __init ptlock_cache_init(void);

extern bool ptlock_alloc(struct page *page);

extern bool ptlock_alloc(struct page *page);

extern void ptlock_free(struct page *page);

extern void ptlock_free(struct page *page);
@@ -1358,6 +1359,10 @@ static inline spinlock_t *ptlock_ptr(struct page *page) 
	return page->ptl;

	return page->ptl;

}

}

#else /* ALLOC_SPLIT_PTLOCKS */

#else /* ALLOC_SPLIT_PTLOCKS */

static inline void ptlock_cache_init(void)

{

}



static inline bool ptlock_alloc(struct page *page)

static inline bool ptlock_alloc(struct page *page)

{

{

	return true;

	return true;
@@ -1410,10 +1415,17 @@ static inline spinlock_t *pte_lockptr(struct mm_struct *mm, pmd_t *pmd) 
{

{

	return &mm->page_table_lock;

	return &mm->page_table_lock;

}

}

static inline void ptlock_cache_init(void) {}

static inline bool ptlock_init(struct page *page) { return true; }

static inline bool ptlock_init(struct page *page) { return true; }

static inline void pte_lock_deinit(struct page *page) {}

static inline void pte_lock_deinit(struct page *page) {}

#endif /* USE_SPLIT_PTE_PTLOCKS */

#endif /* USE_SPLIT_PTE_PTLOCKS */





static inline void pgtable_init(void)

{

	ptlock_cache_init();

	pgtable_cache_init();

}



static inline bool pgtable_page_ctor(struct page *page)

static inline bool pgtable_page_ctor(struct page *page)

{

{

	inc_zone_page_state(page, NR_PAGETABLE);

	inc_zone_page_state(page, NR_PAGETABLE);

init/main.c

+1 −1
Original line number Original line Diff line number Diff line
@@ -476,7 +476,7 @@ static void __init mm_init(void) 
	mem_init();

	mem_init();

	kmem_cache_init();

	kmem_cache_init();

	percpu_init_late();

	percpu_init_late();

	pgtable_cache_init();

	pgtable_init();

	vmalloc_init();

	vmalloc_init();

}

}

mm/memory.c

+11 −2
Original line number Original line Diff line number Diff line
@@ -4275,11 +4275,20 @@ void copy_user_huge_page(struct page *dst, struct page *src, 
#endif /* CONFIG_TRANSPARENT_HUGEPAGE || CONFIG_HUGETLBFS */

#endif /* CONFIG_TRANSPARENT_HUGEPAGE || CONFIG_HUGETLBFS */





#if USE_SPLIT_PTE_PTLOCKS && ALLOC_SPLIT_PTLOCKS

#if USE_SPLIT_PTE_PTLOCKS && ALLOC_SPLIT_PTLOCKS



static struct kmem_cache *page_ptl_cachep;



void __init ptlock_cache_init(void)

{

	page_ptl_cachep = kmem_cache_create("page->ptl", sizeof(spinlock_t), 0,

			SLAB_PANIC, NULL);

}



bool ptlock_alloc(struct page *page)

bool ptlock_alloc(struct page *page)

{

{

	spinlock_t *ptl;

	spinlock_t *ptl;





	ptl = kmalloc(sizeof(spinlock_t), GFP_KERNEL);

	ptl = kmem_cache_alloc(page_ptl_cachep, GFP_KERNEL);

	if (!ptl)

	if (!ptl)

		return false;

		return false;

	page->ptl = ptl;

	page->ptl = ptl;
@@ -4288,6 +4297,6 @@ bool ptlock_alloc(struct page *page) 




void ptlock_free(struct page *page)

void ptlock_free(struct page *page)

{

{

	kfree(page->ptl);

	kmem_cache_free(page_ptl_cachep, page->ptl);

}

}

#endif
 
#endif

CRA Git | Maintained and supported by SUSTech CRA and CCSE