Commit f1e61557 authored by Kirill A. Shutemov's avatar Kirill A. Shutemov Committed by Linus Torvalds
Browse files

mm: pack compound_dtor and compound_order into one word in struct page 



The patch halves space occupied by compound_dtor and compound_order in
struct page.

For compound_order, it's trivial long -> short conversion.

For get_compound_page_dtor(), we now use hardcoded table for destructor
lookup and store its index in the struct page instead of direct pointer
to destructor. It shouldn't be a big trouble to maintain the table: we
have only two destructor and NULL currently.

This patch free up one word in tail pages for reuse. This is preparation
for the next patch.

Signed-off-by: default avatarKirill A. Shutemov <kirill.shutemov@linux.intel.com>
Reviewed-by: default avatarMichal Hocko <mhocko@suse.com>
Acked-by: default avatarVlastimil Babka <vbabka@suse.cz>
Reviewed-by: default avatarAndrea Arcangeli <aarcange@redhat.com>
Cc: "Paul E. McKenney" <paulmck@linux.vnet.ibm.com>
Cc: Andi Kleen <ak@linux.intel.com>
Cc: Aneesh Kumar K.V <aneesh.kumar@linux.vnet.ibm.com>
Cc: Christoph Lameter <cl@linux.com>
Cc: David Rientjes <rientjes@google.com>
Cc: Hugh Dickins <hughd@google.com>
Cc: Joonsoo Kim <iamjoonsoo.kim@lge.com>
Cc: Sergey Senozhatsky <sergey.senozhatsky@gmail.com>
Signed-off-by: default avatarAndrew Morton <akpm@linux-foundation.org>
Signed-off-by: default avatarLinus Torvalds <torvalds@linux-foundation.org>
parent 32e7ba1e

include/linux/mm.h

+19 −5
Original line number Diff line number Diff line
@@ -568,18 +568,32 @@ int split_free_page(struct page *page); 
/*

 * Compound pages have a destructor function.  Provide a

 * prototype for that function and accessor functions.

 * These are _only_ valid on the head of a PG_compound page.

 * These are _only_ valid on the head of a compound page.

 */

typedef void compound_page_dtor(struct page *);



/* Keep the enum in sync with compound_page_dtors array in mm/page_alloc.c */

enum compound_dtor_id {

	NULL_COMPOUND_DTOR,

	COMPOUND_PAGE_DTOR,

#ifdef CONFIG_HUGETLB_PAGE

	HUGETLB_PAGE_DTOR,

#endif

	NR_COMPOUND_DTORS,

};

extern compound_page_dtor * const compound_page_dtors[];



static inline void set_compound_page_dtor(struct page *page,

						compound_page_dtor *dtor)

		enum compound_dtor_id compound_dtor)

{

	page[1].compound_dtor = dtor;

	VM_BUG_ON_PAGE(compound_dtor >= NR_COMPOUND_DTORS, page);

	page[1].compound_dtor = compound_dtor;

}



static inline compound_page_dtor *get_compound_page_dtor(struct page *page)

{

	return page[1].compound_dtor;

	VM_BUG_ON_PAGE(page[1].compound_dtor >= NR_COMPOUND_DTORS, page);

	return compound_page_dtors[page[1].compound_dtor];

}



static inline int compound_order(struct page *page)
@@ -589,7 +603,7 @@ static inline int compound_order(struct page *page) 
	return page[1].compound_order;

}



static inline void set_compound_order(struct page *page, unsigned long order)

static inline void set_compound_order(struct page *page, unsigned int order)

{

	page[1].compound_order = order;

}

include/linux/mm_types.h

+2 −4
Original line number Diff line number Diff line
@@ -28,8 +28,6 @@ struct mem_cgroup; 
		IS_ENABLED(CONFIG_ARCH_ENABLE_SPLIT_PMD_PTLOCK))

#define ALLOC_SPLIT_PTLOCKS	(SPINLOCK_SIZE > BITS_PER_LONG/8)



typedef void compound_page_dtor(struct page *);



/*

 * Each physical page in the system has a struct page associated with

 * it to keep track of whatever it is we are using the page for at the
@@ -136,8 +134,8 @@ struct page { 
						 */

		/* First tail page of compound page */

		struct {

			compound_page_dtor *compound_dtor;

			unsigned long compound_order;

			unsigned short int compound_dtor;

			unsigned short int compound_order;

		};



#if defined(CONFIG_TRANSPARENT_HUGEPAGE) && USE_SPLIT_PMD_PTLOCKS

mm/hugetlb.c

+4 −4
Original line number Diff line number Diff line
@@ -1146,7 +1146,7 @@ static void update_and_free_page(struct hstate *h, struct page *page) 
				1 << PG_writeback);

	}

	VM_BUG_ON_PAGE(hugetlb_cgroup_from_page(page), page);

	set_compound_page_dtor(page, NULL);

	set_compound_page_dtor(page, NULL_COMPOUND_DTOR);

	set_page_refcounted(page);

	if (hstate_is_gigantic(h)) {

		destroy_compound_gigantic_page(page, huge_page_order(h));
@@ -1242,7 +1242,7 @@ void free_huge_page(struct page *page) 
static void prep_new_huge_page(struct hstate *h, struct page *page, int nid)

{

	INIT_LIST_HEAD(&page->lru);

	set_compound_page_dtor(page, free_huge_page);

	set_compound_page_dtor(page, HUGETLB_PAGE_DTOR);

	spin_lock(&hugetlb_lock);

	set_hugetlb_cgroup(page, NULL);

	h->nr_huge_pages++;
@@ -1294,7 +1294,7 @@ int PageHuge(struct page *page) 
		return 0;



	page = compound_head(page);

	return get_compound_page_dtor(page) == free_huge_page;

	return page[1].compound_dtor == HUGETLB_PAGE_DTOR;

}

EXPORT_SYMBOL_GPL(PageHuge);
@@ -1568,7 +1568,7 @@ static struct page *__alloc_buddy_huge_page(struct hstate *h, 
	if (page) {

		INIT_LIST_HEAD(&page->lru);

		r_nid = page_to_nid(page);

		set_compound_page_dtor(page, free_huge_page);

		set_compound_page_dtor(page, HUGETLB_PAGE_DTOR);

		set_hugetlb_cgroup(page, NULL);

		/*

		 * We incremented the global counters already

mm/page_alloc.c

+10 −1
Original line number Diff line number Diff line
@@ -229,6 +229,15 @@ static char * const zone_names[MAX_NR_ZONES] = { 
#endif

};



static void free_compound_page(struct page *page);

compound_page_dtor * const compound_page_dtors[] = {

	NULL,

	free_compound_page,

#ifdef CONFIG_HUGETLB_PAGE

	free_huge_page,

#endif

};



int min_free_kbytes = 1024;

int user_min_free_kbytes = -1;
@@ -458,7 +467,7 @@ void prep_compound_page(struct page *page, unsigned long order) 
	int i;

	int nr_pages = 1 << order;



	set_compound_page_dtor(page, free_compound_page);

	set_compound_page_dtor(page, COMPOUND_PAGE_DTOR);

	set_compound_order(page, order);

	__SetPageHead(page);

	for (i = 1; i < nr_pages; i++) {

CRA Git | Maintained and supported by SUSTech CRA and CCSE