Commit 18229df5 authored by Andy Whitcroft's avatar Andy Whitcroft Committed by Linus Torvalds
Browse files

hugetlb: pull gigantic page initialisation out of the default path 



As we can determine exactly when a gigantic page is in use we can optimise
the common regular page cases by pulling out gigantic page initialisation
into its own function.  As gigantic pages are never released to buddy we
do not need a destructor.  This effectivly reverts the previous change to
the main buddy allocator.  It also adds a paranoid check to ensure we
never release gigantic pages from hugetlbfs to the main buddy.

Signed-off-by: default avatarAndy Whitcroft <apw@shadowen.org>
Cc: Jon Tollefson <kniht@linux.vnet.ibm.com>
Cc: Mel Gorman <mel@csn.ul.ie>
Cc: Nick Piggin <nickpiggin@yahoo.com.au>
Cc: Christoph Lameter <cl@linux-foundation.org>
Cc: <stable@kernel.org>		[2.6.27.x]
Signed-off-by: default avatarAndrew Morton <akpm@linux-foundation.org>
Signed-off-by: default avatarLinus Torvalds <torvalds@linux-foundation.org>
parent 69d177c2

mm/hugetlb.c

+11 −1
Original line number Diff line number Diff line
@@ -491,6 +491,8 @@ static void update_and_free_page(struct hstate *h, struct page *page) 
{

	int i;



	VM_BUG_ON(h->order >= MAX_ORDER);



	h->nr_huge_pages--;

	h->nr_huge_pages_node[page_to_nid(page)]--;

	for (i = 0; i < pages_per_huge_page(h); i++) {
@@ -1005,6 +1007,14 @@ found: 
	return 1;

}



static void prep_compound_huge_page(struct page *page, int order)

{

	if (unlikely(order > (MAX_ORDER - 1)))

		prep_compound_gigantic_page(page, order);

	else

		prep_compound_page(page, order);

}



/* Put bootmem huge pages into the standard lists after mem_map is up */

static void __init gather_bootmem_prealloc(void)

{
@@ -1015,7 +1025,7 @@ static void __init gather_bootmem_prealloc(void) 
		struct hstate *h = m->hstate;

		__ClearPageReserved(page);

		WARN_ON(page_count(page) != 1);

		prep_compound_page(page, h->order);

		prep_compound_huge_page(page, h->order);

		prep_new_huge_page(h, page, page_to_nid(page));

	}

}

mm/internal.h

+1 −0
Original line number Diff line number Diff line
@@ -17,6 +17,7 @@ void free_pgtables(struct mmu_gather *tlb, struct vm_area_struct *start_vma, 
		unsigned long floor, unsigned long ceiling);



extern void prep_compound_page(struct page *page, unsigned long order);

extern void prep_compound_gigantic_page(struct page *page, unsigned long order);



static inline void set_page_count(struct page *page, int v)

{

mm/page_alloc.c

+21 −7
Original line number Diff line number Diff line
@@ -260,6 +260,23 @@ static void free_compound_page(struct page *page) 
}



void prep_compound_page(struct page *page, unsigned long order)

{

	int i;

	int nr_pages = 1 << order;



	set_compound_page_dtor(page, free_compound_page);

	set_compound_order(page, order);

	__SetPageHead(page);

	for (i = 1; i < nr_pages; i++) {

		struct page *p = page + i;



		__SetPageTail(p);

		p->first_page = page;

	}

}



#ifdef CONFIG_HUGETLBFS

void prep_compound_gigantic_page(struct page *page, unsigned long order)

{

	int i;

	int nr_pages = 1 << order;
@@ -268,19 +285,17 @@ void prep_compound_page(struct page *page, unsigned long order) 
	set_compound_page_dtor(page, free_compound_page);

	set_compound_order(page, order);

	__SetPageHead(page);

	for (i = 1; i < nr_pages; i++, p++) {

		if (unlikely((i & (MAX_ORDER_NR_PAGES - 1)) == 0))

			p = pfn_to_page(page_to_pfn(page) + i);

	for (i = 1; i < nr_pages; i++, p = mem_map_next(p, page, i)) {

		__SetPageTail(p);

		p->first_page = page;

	}

}

#endif



static void destroy_compound_page(struct page *page, unsigned long order)

{

	int i;

	int nr_pages = 1 << order;

	struct page *p = page + 1;



	if (unlikely(compound_order(page) != order))

		bad_page(page);
@@ -288,9 +303,8 @@ static void destroy_compound_page(struct page *page, unsigned long order) 
	if (unlikely(!PageHead(page)))

			bad_page(page);

	__ClearPageHead(page);

	for (i = 1; i < nr_pages; i++, p++) {

		if (unlikely((i & (MAX_ORDER_NR_PAGES - 1)) == 0))

			p = pfn_to_page(page_to_pfn(page) + i);

	for (i = 1; i < nr_pages; i++) {

		struct page *p = page + i;



		if (unlikely(!PageTail(p) |

				(p->first_page != page)))

CRA Git | Maintained and supported by SUSTech CRA and CCSE