Commit dab4ead1 authored by Vlastimil Babka's avatar Vlastimil Babka Committed by Linus Torvalds
Browse files

mm, page_owner: make init_pages_in_zone() faster 

In init_pages_in_zone() we currently use the generic set_page_owner()
function to initialize page_owner info for early allocated pages.  This
means we needlessly do lookup_page_ext() twice for each page, and more
importantly save_stack(), which has to unwind the stack and find the
corresponding stack depot handle.  Because the stack is always the same
for the initialization, unwind it once in init_pages_in_zone() and reuse
the handle.  Also avoid the repeated lookup_page_ext().

This can significantly reduce boot times with page_owner=on on large
machines, especially for kernels built without frame pointer, where the
stack unwinding is noticeably slower.

[vbabka@suse.cz: don't duplicate code of __set_page_owner(), per Michal Hocko]
[akpm@linux-foundation.org: coding-style fixes]
[vbabka@suse.cz: create statically allocated fake stack trace for early allocated pages, per Michal]
  Link: http://lkml.kernel.org/r/45813564-2342-fc8d-d31a-f4b68a724325@suse.cz
Link: http://lkml.kernel.org/r/20170720134029.25268-2-vbabka@suse.cz


Signed-off-by: default avatarVlastimil Babka <vbabka@suse.cz>
Acked-by: default avatarMichal Hocko <mhocko@suse.com>
Cc: Joonsoo Kim <iamjoonsoo.kim@lge.com>
Cc: Mel Gorman <mgorman@techsingularity.net>
Cc: Yang Shi <yang.shi@linaro.org>
Cc: Laura Abbott <labbott@redhat.com>
Cc: Vinayak Menon <vinmenon@codeaurora.org>
Cc: zhong jiang <zhongjiang@huawei.com>
Signed-off-by: default avatarAndrew Morton <akpm@linux-foundation.org>
Signed-off-by: default avatarLinus Torvalds <torvalds@linux-foundation.org>
parent b95046b0

mm/page_owner.c

+32 −20
Original line number Diff line number Diff line
@@ -30,6 +30,7 @@ DEFINE_STATIC_KEY_FALSE(page_owner_inited); 


static depot_stack_handle_t dummy_handle;

static depot_stack_handle_t failure_handle;

static depot_stack_handle_t early_handle;



static void init_early_allocated_pages(void);
@@ -53,7 +54,7 @@ static bool need_page_owner(void) 
	return true;

}



static noinline void register_dummy_stack(void)

static __always_inline depot_stack_handle_t create_dummy_stack(void)

{

	unsigned long entries[4];

	struct stack_trace dummy;
@@ -64,21 +65,22 @@ static noinline void register_dummy_stack(void) 
	dummy.skip = 0;



	save_stack_trace(&dummy);

	dummy_handle = depot_save_stack(&dummy, GFP_KERNEL);

	return depot_save_stack(&dummy, GFP_KERNEL);

}



static noinline void register_failure_stack(void)

static noinline void register_dummy_stack(void)

{

	unsigned long entries[4];

	struct stack_trace failure;

	dummy_handle = create_dummy_stack();

}



	failure.nr_entries = 0;

	failure.max_entries = ARRAY_SIZE(entries);

	failure.entries = &entries[0];

	failure.skip = 0;

static noinline void register_failure_stack(void)

{

	failure_handle = create_dummy_stack();

}



	save_stack_trace(&failure);

	failure_handle = depot_save_stack(&failure, GFP_KERNEL);

static noinline void register_early_stack(void)

{

	early_handle = create_dummy_stack();

}



static void init_page_owner(void)
@@ -88,6 +90,7 @@ static void init_page_owner(void) 


	register_dummy_stack();

	register_failure_stack();

	register_early_stack();

	static_branch_enable(&page_owner_inited);

	init_early_allocated_pages();

}
@@ -165,17 +168,13 @@ static noinline depot_stack_handle_t save_stack(gfp_t flags) 
	return handle;

}



noinline void __set_page_owner(struct page *page, unsigned int order,

					gfp_t gfp_mask)

static inline void __set_page_owner_handle(struct page_ext *page_ext,

	depot_stack_handle_t handle, unsigned int order, gfp_t gfp_mask)

{

	struct page_ext *page_ext = lookup_page_ext(page);

	struct page_owner *page_owner;



	if (unlikely(!page_ext))

		return;



	page_owner = get_page_owner(page_ext);

	page_owner->handle = save_stack(gfp_mask);

	page_owner->handle = handle;

	page_owner->order = order;

	page_owner->gfp_mask = gfp_mask;

	page_owner->last_migrate_reason = -1;
@@ -183,6 +182,19 @@ noinline void __set_page_owner(struct page *page, unsigned int order, 
	__set_bit(PAGE_EXT_OWNER, &page_ext->flags);

}



noinline void __set_page_owner(struct page *page, unsigned int order,

					gfp_t gfp_mask)

{

	struct page_ext *page_ext = lookup_page_ext(page);

	depot_stack_handle_t handle;



	if (unlikely(!page_ext))

		return;



	handle = save_stack(gfp_mask);

	__set_page_owner_handle(page_ext, handle, order, gfp_mask);

}



void __set_page_owner_migrate_reason(struct page *page, int reason)

{

	struct page_ext *page_ext = lookup_page_ext(page);
@@ -565,12 +577,12 @@ static void init_pages_in_zone(pg_data_t *pgdat, struct zone *zone) 
			if (unlikely(!page_ext))

				continue;



			/* Maybe overraping zone */

			/* Maybe overlapping zone */

			if (test_bit(PAGE_EXT_OWNER, &page_ext->flags))

				continue;



			/* Found early allocated page */

			set_page_owner(page, 0, 0);

			__set_page_owner_handle(page_ext, early_handle, 0, 0);

			count++;

		}

	}

CRA Git | Maintained and supported by SUSTech CRA and CCSE