Commit c65e774f authored by Kirill A. Shutemov's avatar Kirill A. Shutemov Committed by Ingo Molnar
Browse files

x86/mm: Make PGDIR_SHIFT and PTRS_PER_P4D variable 



For boot-time switching between 4- and 5-level paging we need to be able
to fold p4d page table level at runtime. It requires variable
PGDIR_SHIFT and PTRS_PER_P4D.

The change doesn't affect the kernel image size much:

   text	   data	    bss	    dec	    hex	filename
8628091	4734304	1368064	14730459	 e0c4db	vmlinux.before
8628393	4734340	1368064	14730797	 e0c62d	vmlinux.after

Signed-off-by: default avatarKirill A. Shutemov <kirill.shutemov@linux.intel.com>
Cc: Andy Lutomirski <luto@amacapital.net>
Cc: Borislav Petkov <bp@suse.de>
Cc: Linus Torvalds <torvalds@linux-foundation.org>
Cc: Peter Zijlstra <peterz@infradead.org>
Cc: Thomas Gleixner <tglx@linutronix.de>
Cc: linux-mm@kvack.org
Link: http://lkml.kernel.org/r/20180214111656.88514-7-kirill.shutemov@linux.intel.com


Signed-off-by: default avatarIngo Molnar <mingo@kernel.org>
parent 5c7919bb

arch/x86/boot/compressed/kaslr.c

+2 −0
Original line number Diff line number Diff line
@@ -48,6 +48,8 @@ 


#ifdef CONFIG_X86_5LEVEL

unsigned int pgtable_l5_enabled __ro_after_init = 1;

unsigned int pgdir_shift __ro_after_init = 48;

unsigned int ptrs_per_p4d __ro_after_init = 512;

#endif



extern unsigned long get_cmd_line_ptr(void);

arch/x86/include/asm/pgtable_32.h

+2 −0
Original line number Diff line number Diff line
@@ -33,6 +33,8 @@ static inline void pgtable_cache_init(void) { } 
static inline void check_pgt_cache(void) { }

void paging_init(void);



static inline int pgd_large(pgd_t pgd) { return 0; }



/*

 * Define this if things work differently on an i386 and an i486:

 * it will (on an i486) warn about kernel memory accesses that are

arch/x86/include/asm/pgtable_64_types.h

+12 −7
Original line number Diff line number Diff line
@@ -26,6 +26,9 @@ extern unsigned int pgtable_l5_enabled; 
#define pgtable_l5_enabled 0

#endif



extern unsigned int pgdir_shift;

extern unsigned int ptrs_per_p4d;



#endif	/* !__ASSEMBLY__ */



#define SHARED_KERNEL_PMD	0
@@ -35,14 +38,15 @@ extern unsigned int pgtable_l5_enabled; 
/*

 * PGDIR_SHIFT determines what a top-level page table entry can map

 */

#define PGDIR_SHIFT	48

#define PGDIR_SHIFT	pgdir_shift

#define PTRS_PER_PGD	512



/*

 * 4th level page in 5-level paging case

 */

#define P4D_SHIFT		39

#define PTRS_PER_P4D	512

#define MAX_PTRS_PER_P4D	512

#define PTRS_PER_P4D		ptrs_per_p4d

#define P4D_SIZE		(_AC(1, UL) << P4D_SHIFT)

#define P4D_MASK		(~(P4D_SIZE - 1))
@@ -55,6 +59,7 @@ extern unsigned int pgtable_l5_enabled; 
 */

#define PGDIR_SHIFT		39

#define PTRS_PER_PGD		512

#define MAX_PTRS_PER_P4D	1



#endif /* CONFIG_X86_5LEVEL */

arch/x86/kernel/cpu/mcheck/mce.c

+6 −12
Original line number Diff line number Diff line
@@ -1082,19 +1082,7 @@ void arch_unmap_kpfn(unsigned long pfn) 
	 * a legal address.

	 */



/*

 * Build time check to see if we have a spare virtual bit. Don't want

 * to leave this until run time because most developers don't have a

 * system that can exercise this code path. This will only become a

 * problem if/when we move beyond 5-level page tables.

 *

 * Hard code "9" here because cpp doesn't grok ilog2(PTRS_PER_PGD)

 */

#if PGDIR_SHIFT + 9 < 63

	decoy_addr = (pfn << PAGE_SHIFT) + (PAGE_OFFSET ^ BIT(63));

#else

#error "no unused virtual bit available"

#endif



	if (set_memory_np(decoy_addr, 1))

		pr_warn("Could not invalidate pfn=0x%lx from 1:1 map\n", pfn);
@@ -2328,6 +2316,12 @@ static __init int mcheck_init_device(void) 
{

	int err;



	/*

	 * Check if we have a spare virtual bit. This will only become

	 * a problem if/when we move beyond 5-level page tables.

	 */

	MAYBE_BUILD_BUG_ON(__VIRTUAL_MASK_SHIFT >= 63);



	if (!mce_available(&boot_cpu_data)) {

		err = -EIO;

		goto err_out;

arch/x86/kernel/head64.c

+5 −1
Original line number Diff line number Diff line
@@ -42,6 +42,10 @@ pmdval_t early_pmd_flags = __PAGE_KERNEL_LARGE & ~(_PAGE_GLOBAL | _PAGE_NX); 
#ifdef CONFIG_X86_5LEVEL

unsigned int pgtable_l5_enabled __ro_after_init = 1;

EXPORT_SYMBOL(pgtable_l5_enabled);

unsigned int pgdir_shift __ro_after_init = 48;

EXPORT_SYMBOL(pgdir_shift);

unsigned int ptrs_per_p4d __ro_after_init = 512;

EXPORT_SYMBOL(ptrs_per_p4d);

#endif



#ifdef CONFIG_DYNAMIC_MEMORY_LAYOUT
@@ -336,7 +340,7 @@ asmlinkage __visible void __init x86_64_start_kernel(char * real_mode_data) 
	BUILD_BUG_ON((__START_KERNEL_map & ~PMD_MASK) != 0);

	BUILD_BUG_ON((MODULES_VADDR & ~PMD_MASK) != 0);

	BUILD_BUG_ON(!(MODULES_VADDR > __START_KERNEL));

	BUILD_BUG_ON(!(((MODULES_END - 1) & PGDIR_MASK) ==

	MAYBE_BUILD_BUG_ON(!(((MODULES_END - 1) & PGDIR_MASK) ==

				(__START_KERNEL & PGDIR_MASK)));

	BUILD_BUG_ON(__fix_to_virt(__end_of_fixed_addresses) <= MODULES_END);

CRA Git | Maintained and supported by SUSTech CRA and CCSE