Commit 04013513 authored by Vlastimil Babka's avatar Vlastimil Babka Committed by Linus Torvalds
Browse files

mm, page_alloc: do not rely on the order of page_poison and init_on_alloc/free parameters 

Patch series "cleanup page poisoning", v3.

I have identified a number of issues and opportunities for cleanup with
CONFIG_PAGE_POISON and friends:

 - interaction with init_on_alloc and init_on_free parameters depends on
   the order of parameters (Patch 1)

 - the boot time enabling uses static key, but inefficienty (Patch 2)

 - sanity checking is incompatible with hibernation (Patch 3)

 - CONFIG_PAGE_POISONING_NO_SANITY can be removed now that we have
   init_on_free (Patch 4)

 - CONFIG_PAGE_POISONING_ZERO can be most likely removed now that we
   have init_on_free (Patch 5)

This patch (of 5):

Enabling page_poison=1 together with init_on_alloc=1 or init_on_free=1
produces a warning in dmesg that page_poison takes precedence.  However,
as these warnings are printed in early_param handlers for
init_on_alloc/free, they are not printed if page_poison is enabled later
on the command line (handlers are called in the order of their
parameters), or when init_on_alloc/free is always enabled by the
respective config option - before the page_poison early param handler is
called, it is not considered to be enabled.  This is inconsistent.

We can remove the dependency on order by making the init_on_* parameters
only set a boolean variable, and postponing the evaluation after all early
params have been processed.  Introduce a new
init_mem_debugging_and_hardening() function for that, and move the related
debug_pagealloc processing there as well.

As a result init_mem_debugging_and_hardening() knows always accurately if
init_on_* and/or page_poison options were enabled.  Thus we can also
optimize want_init_on_alloc() and want_init_on_free().  We don't need to
check page_poisoning_enabled() there, we can instead not enable the
init_on_* static keys at all, if page poisoning is enabled.  This results
in a simpler and more effective code.

Link: https://lkml.kernel.org/r/20201113104033.22907-1-vbabka@suse.cz
Link: https://lkml.kernel.org/r/20201113104033.22907-2-vbabka@suse.cz


Signed-off-by: default avatarVlastimil Babka <vbabka@suse.cz>
Reviewed-by: default avatarDavid Hildenbrand <david@redhat.com>
Reviewed-by: default avatarMike Rapoport <rppt@linux.ibm.com>
Cc: Rafael J. Wysocki <rafael.j.wysocki@intel.com>
Cc: Alexander Potapenko <glider@google.com>
Cc: Kees Cook <keescook@chromium.org>
Cc: Michal Hocko <mhocko@kernel.org>
Cc: Mateusz Nosek <mateusznosek0@gmail.com>
Cc: Laura Abbott <labbott@kernel.org>
Signed-off-by: default avatarAndrew Morton <akpm@linux-foundation.org>
Signed-off-by: default avatarLinus Torvalds <torvalds@linux-foundation.org>
parent b8ca396f

include/linux/mm.h

+3 −17
Original line number Diff line number Diff line
@@ -2872,6 +2872,7 @@ extern int apply_to_existing_page_range(struct mm_struct *mm, 
				   unsigned long address, unsigned long size,

				   pte_fn_t fn, void *data);



extern void init_mem_debugging_and_hardening(void);

#ifdef CONFIG_PAGE_POISONING

extern bool page_poisoning_enabled(void);

extern void kernel_poison_pages(struct page *page, int numpages, int enable);
@@ -2881,35 +2882,20 @@ static inline void kernel_poison_pages(struct page *page, int numpages, 
					int enable) { }

#endif



#ifdef CONFIG_INIT_ON_ALLOC_DEFAULT_ON

DECLARE_STATIC_KEY_TRUE(init_on_alloc);

#else

DECLARE_STATIC_KEY_FALSE(init_on_alloc);

#endif

static inline bool want_init_on_alloc(gfp_t flags)

{

	if (static_branch_unlikely(&init_on_alloc) &&

	    !page_poisoning_enabled())

	if (static_branch_unlikely(&init_on_alloc))

		return true;

	return flags & __GFP_ZERO;

}



#ifdef CONFIG_INIT_ON_FREE_DEFAULT_ON

DECLARE_STATIC_KEY_TRUE(init_on_free);

#else

DECLARE_STATIC_KEY_FALSE(init_on_free);

#endif

static inline bool want_init_on_free(void)

{

	return static_branch_unlikely(&init_on_free) &&

	       !page_poisoning_enabled();

	return static_branch_unlikely(&init_on_free);

}



#ifdef CONFIG_DEBUG_PAGEALLOC

extern void init_debug_pagealloc(void);

#else

static inline void init_debug_pagealloc(void) {}

#endif

extern bool _debug_pagealloc_enabled_early;

DECLARE_STATIC_KEY_FALSE(_debug_pagealloc_enabled);

init/main.c

+1 −1
Original line number Diff line number Diff line
@@ -824,7 +824,7 @@ static void __init mm_init(void) 
	 * bigger than MAX_ORDER unless SPARSEMEM.

	 */

	page_ext_init_flatmem();

	init_debug_pagealloc();

	init_mem_debugging_and_hardening();

	report_meminit();

	mem_init();

	/* page_owner must be initialized after buddy is ready */

mm/page_alloc.c

+42 −46
Original line number Diff line number Diff line
@@ -167,53 +167,26 @@ unsigned long totalcma_pages __read_mostly; 


int percpu_pagelist_fraction;

gfp_t gfp_allowed_mask __read_mostly = GFP_BOOT_MASK;

#ifdef CONFIG_INIT_ON_ALLOC_DEFAULT_ON

DEFINE_STATIC_KEY_TRUE(init_on_alloc);

#else

DEFINE_STATIC_KEY_FALSE(init_on_alloc);

#endif

EXPORT_SYMBOL(init_on_alloc);



#ifdef CONFIG_INIT_ON_FREE_DEFAULT_ON

DEFINE_STATIC_KEY_TRUE(init_on_free);

#else

DEFINE_STATIC_KEY_FALSE(init_on_free);

#endif

EXPORT_SYMBOL(init_on_free);



static bool _init_on_alloc_enabled_early __read_mostly

				= IS_ENABLED(CONFIG_INIT_ON_ALLOC_DEFAULT_ON);

static int __init early_init_on_alloc(char *buf)

{

	int ret;

	bool bool_result;



	ret = kstrtobool(buf, &bool_result);

	if (ret)

		return ret;

	if (bool_result && page_poisoning_enabled())

		pr_info("mem auto-init: CONFIG_PAGE_POISONING is on, will take precedence over init_on_alloc\n");

	if (bool_result)

		static_branch_enable(&init_on_alloc);

	else

		static_branch_disable(&init_on_alloc);

	return 0;

	return kstrtobool(buf, &_init_on_alloc_enabled_early);

}

early_param("init_on_alloc", early_init_on_alloc);



static bool _init_on_free_enabled_early __read_mostly

				= IS_ENABLED(CONFIG_INIT_ON_FREE_DEFAULT_ON);

static int __init early_init_on_free(char *buf)

{

	int ret;

	bool bool_result;



	ret = kstrtobool(buf, &bool_result);

	if (ret)

		return ret;

	if (bool_result && page_poisoning_enabled())

		pr_info("mem auto-init: CONFIG_PAGE_POISONING is on, will take precedence over init_on_free\n");

	if (bool_result)

		static_branch_enable(&init_on_free);

	else

		static_branch_disable(&init_on_free);

	return 0;

	return kstrtobool(buf, &_init_on_free_enabled_early);

}

early_param("init_on_free", early_init_on_free);
@@ -730,19 +703,6 @@ static int __init early_debug_pagealloc(char *buf) 
}

early_param("debug_pagealloc", early_debug_pagealloc);



void init_debug_pagealloc(void)

{

	if (!debug_pagealloc_enabled())

		return;



	static_branch_enable(&_debug_pagealloc_enabled);



	if (!debug_guardpage_minorder())

		return;



	static_branch_enable(&_debug_guardpage_enabled);

}



static int __init debug_guardpage_minorder_setup(char *buf)

{

	unsigned long res;
@@ -794,6 +754,42 @@ static inline void clear_page_guard(struct zone *zone, struct page *page, 
				unsigned int order, int migratetype) {}

#endif



/*

 * Enable static keys related to various memory debugging and hardening options.

 * Some override others, and depend on early params that are evaluated in the

 * order of appearance. So we need to first gather the full picture of what was

 * enabled, and then make decisions.

 */

void init_mem_debugging_and_hardening(void)

{

	if (_init_on_alloc_enabled_early) {

		if (page_poisoning_enabled())

			pr_info("mem auto-init: CONFIG_PAGE_POISONING is on, "

				"will take precedence over init_on_alloc\n");

		else

			static_branch_enable(&init_on_alloc);

	}

	if (_init_on_free_enabled_early) {

		if (page_poisoning_enabled())

			pr_info("mem auto-init: CONFIG_PAGE_POISONING is on, "

				"will take precedence over init_on_free\n");

		else

			static_branch_enable(&init_on_free);

	}



#ifdef CONFIG_DEBUG_PAGEALLOC

	if (!debug_pagealloc_enabled())

		return;



	static_branch_enable(&_debug_pagealloc_enabled);



	if (!debug_guardpage_minorder())

		return;



	static_branch_enable(&_debug_guardpage_enabled);

#endif

}



static inline void set_buddy_order(struct page *page, unsigned int order)

{

	set_page_private(page, order);

CRA Git | Maintained and supported by SUSTech CRA and CCSE