Commit 3e1907d5 authored by Ard Biesheuvel's avatar Ard Biesheuvel Committed by Will Deacon
Browse files

arm64: mm: move vmemmap region right below the linear region 



This moves the vmemmap region right below PAGE_OFFSET, aka the start
of the linear region, and redefines its size to be a power of two.
Due to the placement of PAGE_OFFSET in the middle of the address space,
whose size is a power of two as well, this guarantees that virt to
page conversions and vice versa can be implemented efficiently, by
masking and shifting rather than ordinary arithmetic.

Signed-off-by: default avatarArd Biesheuvel <ard.biesheuvel@linaro.org>
Signed-off-by: default avatarWill Deacon <will.deacon@arm.com>
parent e44308e6

arch/arm64/include/asm/memory.h

+17 −1
Original line number Diff line number Diff line
@@ -39,6 +39,21 @@ 
 */

#define PCI_IO_SIZE		SZ_16M



/*

 * Log2 of the upper bound of the size of a struct page. Used for sizing

 * the vmemmap region only, does not affect actual memory footprint.

 * We don't use sizeof(struct page) directly since taking its size here

 * requires its definition to be available at this point in the inclusion

 * chain, and it may not be a power of 2 in the first place.

 */

#define STRUCT_PAGE_MAX_SHIFT	6



/*

 * VMEMMAP_SIZE - allows the whole linear region to be covered by

 *                a struct page array

 */

#define VMEMMAP_SIZE (UL(1) << (VA_BITS - PAGE_SHIFT - 1 + STRUCT_PAGE_MAX_SHIFT))



/*

 * PAGE_OFFSET - the virtual address of the start of the kernel image (top

 *		 (VA_BITS - 1))
@@ -54,7 +69,8 @@ 
#define MODULES_END		(MODULES_VADDR + MODULES_VSIZE)

#define MODULES_VADDR		(VA_START + KASAN_SHADOW_SIZE)

#define MODULES_VSIZE		(SZ_128M)

#define PCI_IO_END		(PAGE_OFFSET - SZ_2M)

#define VMEMMAP_START		(PAGE_OFFSET - VMEMMAP_SIZE)

#define PCI_IO_END		(VMEMMAP_START - SZ_2M)

#define PCI_IO_START		(PCI_IO_END - PCI_IO_SIZE)

#define FIXADDR_TOP		(PCI_IO_START - SZ_2M)

#define TASK_SIZE_64		(UL(1) << VA_BITS)

arch/arm64/include/asm/pgtable.h

+3 −8
Original line number Diff line number Diff line
@@ -24,20 +24,15 @@ 
#include <asm/pgtable-prot.h>



/*

 * VMALLOC and SPARSEMEM_VMEMMAP ranges.

 * VMALLOC range.

 *

 * VMEMAP_SIZE: allows the whole linear region to be covered by a struct page array

 *	(rounded up to PUD_SIZE).

 * VMALLOC_START: beginning of the kernel vmalloc space

 * VMALLOC_END: extends to the available space below vmmemmap, PCI I/O space,

 *	fixed mappings and modules

 * VMALLOC_END: extends to the available space below vmmemmap, PCI I/O space

 *	and fixed mappings

 */

#define VMEMMAP_SIZE		ALIGN((1UL << (VA_BITS - PAGE_SHIFT - 1)) * sizeof(struct page), PUD_SIZE)



#define VMALLOC_START		(MODULES_END)

#define VMALLOC_END		(PAGE_OFFSET - PUD_SIZE - VMEMMAP_SIZE - SZ_64K)



#define VMEMMAP_START		(VMALLOC_END + SZ_64K)

#define vmemmap			((struct page *)VMEMMAP_START - (memstart_addr >> PAGE_SHIFT))



#define FIRST_USER_ADDRESS	0UL

arch/arm64/mm/dump.c

+8 −8
Original line number Diff line number Diff line
@@ -37,14 +37,14 @@ enum address_markers_idx { 
	MODULES_END_NR,

	VMALLOC_START_NR,

	VMALLOC_END_NR,

#ifdef CONFIG_SPARSEMEM_VMEMMAP

	VMEMMAP_START_NR,

	VMEMMAP_END_NR,

#endif

	FIXADDR_START_NR,

	FIXADDR_END_NR,

	PCI_START_NR,

	PCI_END_NR,

#ifdef CONFIG_SPARSEMEM_VMEMMAP

	VMEMMAP_START_NR,

	VMEMMAP_END_NR,

#endif

	KERNEL_SPACE_NR,

};
@@ -53,14 +53,14 @@ static struct addr_marker address_markers[] = { 
	{ MODULES_END,		"Modules end" },

	{ VMALLOC_START,	"vmalloc() Area" },

	{ VMALLOC_END,		"vmalloc() End" },

#ifdef CONFIG_SPARSEMEM_VMEMMAP

	{ 0,			"vmemmap start" },

	{ 0,			"vmemmap end" },

#endif

	{ FIXADDR_START,	"Fixmap start" },

	{ FIXADDR_TOP,		"Fixmap end" },

	{ PCI_IO_START,		"PCI I/O start" },

	{ PCI_IO_END,		"PCI I/O end" },

#ifdef CONFIG_SPARSEMEM_VMEMMAP

	{ 0,			"vmemmap start" },

	{ 0,			"vmemmap end" },

#endif

	{ PAGE_OFFSET,		"Linear Mapping" },

	{ -1,			NULL },

};

arch/arm64/mm/init.c

+10 −4
Original line number Diff line number Diff line
@@ -412,6 +412,10 @@ void __init mem_init(void) 
		MLK_ROUNDUP(__start_rodata, _etext),

		MLK_ROUNDUP(__init_begin, __init_end),

		MLK_ROUNDUP(_sdata, _edata));

	pr_cont("    fixed   : 0x%16lx - 0x%16lx   (%6ld KB)\n",

		MLK(FIXADDR_START, FIXADDR_TOP));

	pr_cont("    PCI I/O : 0x%16lx - 0x%16lx   (%6ld MB)\n",

		MLM(PCI_IO_START, PCI_IO_END));

#ifdef CONFIG_SPARSEMEM_VMEMMAP

	pr_cont("    vmemmap : 0x%16lx - 0x%16lx   (%6ld GB maximum)\n"

		"              0x%16lx - 0x%16lx   (%6ld MB actual)\n",
@@ -420,10 +424,6 @@ void __init mem_init(void) 
		MLM((unsigned long)phys_to_page(memblock_start_of_DRAM()),

		    (unsigned long)virt_to_page(high_memory)));

#endif

	pr_cont("    fixed   : 0x%16lx - 0x%16lx   (%6ld KB)\n",

		MLK(FIXADDR_START, FIXADDR_TOP));

	pr_cont("    PCI I/O : 0x%16lx - 0x%16lx   (%6ld MB)\n",

		MLM(PCI_IO_START, PCI_IO_END));

	pr_cont("    memory  : 0x%16lx - 0x%16lx   (%6ld MB)\n",

		MLM(__phys_to_virt(memblock_start_of_DRAM()),

		    (unsigned long)high_memory));
@@ -440,6 +440,12 @@ void __init mem_init(void) 
	BUILD_BUG_ON(TASK_SIZE_32			> TASK_SIZE_64);

#endif



	/*

	 * Make sure we chose the upper bound of sizeof(struct page)

	 * correctly.

	 */

	BUILD_BUG_ON(sizeof(struct page) > (1 << STRUCT_PAGE_MAX_SHIFT));



	if (PAGE_SIZE >= 16384 && get_num_physpages() <= 128) {

		extern int sysctl_overcommit_memory;

		/*

CRA Git | Maintained and supported by SUSTech CRA and CCSE