Commit c79b954b authored by Jungseok Lee's avatar Jungseok Lee Committed by Catalin Marinas
Browse files

arm64: mm: Implement 4 levels of translation tables 



This patch implements 4 levels of translation tables since 3 levels
of page tables with 4KB pages cannot support 40-bit physical address
space described in [1] due to the following issue.

It is a restriction that kernel logical memory map with 4KB + 3 levels
(0xffffffc000000000-0xffffffffffffffff) cannot cover RAM region from
544GB to 1024GB in [1]. Specifically, ARM64 kernel fails to create
mapping for this region in map_mem function since __phys_to_virt for
this region reaches to address overflow.

If SoC design follows the document, [1], over 32GB RAM would be placed
from 544GB. Even 64GB system is supposed to use the region from 544GB
to 576GB for only 32GB RAM. Naturally, it would reach to enable 4 levels
of page tables to avoid hacking __virt_to_phys and __phys_to_virt.

However, it is recommended 4 levels of page table should be only enabled
if memory map is too sparse or there is about 512GB RAM.

References
----------
[1]: Principles of ARM Memory Maps, White Paper, Issue C

Signed-off-by: default avatarJungseok Lee <jays.lee@samsung.com>
Reviewed-by: default avatarSungjinn Chung <sungjinn.chung@samsung.com>
Acked-by: default avatarKukjin Kim <kgene.kim@samsung.com>
Reviewed-by: default avatarChristoffer Dall <christoffer.dall@linaro.org>
Reviewed-by: default avatarSteve Capper <steve.capper@linaro.org>
[catalin.marinas@arm.com: MEMBLOCK_INITIAL_LIMIT removed, same as PUD_SIZE]
[catalin.marinas@arm.com: early_ioremap_init() updated for 4 levels]
[catalin.marinas@arm.com: 48-bit VA depends on BROKEN until KVM is fixed]
Signed-off-by: default avatarCatalin Marinas <catalin.marinas@arm.com>
Tested-by: default avatarJungseok Lee <jungseoklee85@gmail.com>
parent 57e01390

arch/arm64/Kconfig

+8 −0
Original line number Diff line number Diff line
@@ -195,12 +195,17 @@ config ARM64_VA_BITS_42 
	bool "42-bit"

	depends on ARM64_64K_PAGES



config ARM64_VA_BITS_48

	bool "48-bit"

	depends on BROKEN



endchoice



config ARM64_VA_BITS

	int

	default 39 if ARM64_VA_BITS_39

	default 42 if ARM64_VA_BITS_42

	default 48 if ARM64_VA_BITS_48



config ARM64_2_LEVELS

	def_bool y if ARM64_64K_PAGES && ARM64_VA_BITS_42
@@ -208,6 +213,9 @@ config ARM64_2_LEVELS 
config ARM64_3_LEVELS

	def_bool y if ARM64_4K_PAGES && ARM64_VA_BITS_39



config ARM64_4_LEVELS

	def_bool y if ARM64_4K_PAGES && ARM64_VA_BITS_48



config CPU_BIG_ENDIAN

       bool "Build big-endian kernel"

       help

arch/arm64/include/asm/page.h

+10 −3
Original line number Diff line number Diff line
@@ -33,19 +33,26 @@ 


/*

 * The idmap and swapper page tables need some space reserved in the kernel

 * image. Both require a pgd and a next level table to (section) map the

 * kernel. The the swapper also maps the FDT (see __create_page_tables for

 * image. Both require pgd, pud (4 levels only) and pmd tables to (section)

 * map the kernel. The swapper also maps the FDT (see __create_page_tables for

 * more information).

 */

#ifdef CONFIG_ARM64_4_LEVELS

#define SWAPPER_DIR_SIZE	(3 * PAGE_SIZE)

#define IDMAP_DIR_SIZE		(3 * PAGE_SIZE)

#else

#define SWAPPER_DIR_SIZE	(2 * PAGE_SIZE)

#define IDMAP_DIR_SIZE		(2 * PAGE_SIZE)

#endif



#ifndef __ASSEMBLY__



#ifdef CONFIG_ARM64_2_LEVELS

#include <asm/pgtable-2level-types.h>

#else

#elif defined(CONFIG_ARM64_3_LEVELS)

#include <asm/pgtable-3level-types.h>

#else

#include <asm/pgtable-4level-types.h>

#endif



extern void __cpu_clear_user_page(void *p, unsigned long user);

arch/arm64/include/asm/pgalloc.h

+20 −0
Original line number Diff line number Diff line
@@ -46,6 +46,26 @@ static inline void pud_populate(struct mm_struct *mm, pud_t *pud, pmd_t *pmd) 


#endif	/* CONFIG_ARM64_2_LEVELS */



#ifdef CONFIG_ARM64_4_LEVELS



static inline pud_t *pud_alloc_one(struct mm_struct *mm, unsigned long addr)

{

	return (pud_t *)get_zeroed_page(GFP_KERNEL | __GFP_REPEAT);

}



static inline void pud_free(struct mm_struct *mm, pud_t *pud)

{

	BUG_ON((unsigned long)pud & (PAGE_SIZE-1));

	free_page((unsigned long)pud);

}



static inline void pgd_populate(struct mm_struct *mm, pgd_t *pgd, pud_t *pud)

{

	set_pgd(pgd, __pgd(__pa(pud) | PUD_TYPE_TABLE));

}



#endif	/* CONFIG_ARM64_4_LEVELS */



extern pgd_t *pgd_alloc(struct mm_struct *mm);

extern void pgd_free(struct mm_struct *mm, pgd_t *pgd);

arch/arm64/include/asm/pgtable-hwdef.h

+4 −2
Original line number Diff line number Diff line
@@ -18,8 +18,10 @@ 


#ifdef CONFIG_ARM64_2_LEVELS

#include <asm/pgtable-2level-hwdef.h>

#else

#elif defined(CONFIG_ARM64_3_LEVELS)

#include <asm/pgtable-3level-hwdef.h>

#else

#include <asm/pgtable-4level-hwdef.h>

#endif



/*
@@ -27,7 +29,7 @@ 
 *

 * Level 1 descriptor (PUD).

 */



#define PUD_TYPE_TABLE		(_AT(pudval_t, 3) << 0)

#define PUD_TABLE_BIT		(_AT(pgdval_t, 1) << 1)

#define PUD_TYPE_MASK		(_AT(pgdval_t, 3) << 0)

#define PUD_TYPE_SECT		(_AT(pgdval_t, 1) << 0)

arch/arm64/include/asm/pgtable.h

+40 −0
Original line number Diff line number Diff line
@@ -35,7 +35,11 @@ 
 * VMALLOC and SPARSEMEM_VMEMMAP ranges.

 */

#define VMALLOC_START		(UL(0xffffffffffffffff) << VA_BITS)

#ifndef CONFIG_ARM64_4_LEVELS

#define VMALLOC_END		(PAGE_OFFSET - UL(0x400000000) - SZ_64K)

#else

#define VMALLOC_END		(PAGE_OFFSET - UL(0x40000000000) - SZ_64K)

#endif



#define vmemmap			((struct page *)(VMALLOC_END + SZ_64K))
@@ -44,12 +48,16 @@ 
#ifndef __ASSEMBLY__

extern void __pte_error(const char *file, int line, unsigned long val);

extern void __pmd_error(const char *file, int line, unsigned long val);

extern void __pud_error(const char *file, int line, unsigned long val);

extern void __pgd_error(const char *file, int line, unsigned long val);



#define pte_ERROR(pte)		__pte_error(__FILE__, __LINE__, pte_val(pte))

#ifndef CONFIG_ARM64_2_LEVELS

#define pmd_ERROR(pmd)		__pmd_error(__FILE__, __LINE__, pmd_val(pmd))

#endif

#ifdef CONFIG_ARM64_4_LEVELS

#define pud_ERROR(pud)		__pud_error(__FILE__, __LINE__, pud_val(pud))

#endif

#define pgd_ERROR(pgd)		__pgd_error(__FILE__, __LINE__, pgd_val(pgd))



#ifdef CONFIG_SMP
@@ -347,6 +355,30 @@ static inline pmd_t *pud_page_vaddr(pud_t pud) 


#endif	/* CONFIG_ARM64_2_LEVELS */



#ifdef CONFIG_ARM64_4_LEVELS



#define pgd_none(pgd)		(!pgd_val(pgd))

#define pgd_bad(pgd)		(!(pgd_val(pgd) & 2))

#define pgd_present(pgd)	(pgd_val(pgd))



static inline void set_pgd(pgd_t *pgdp, pgd_t pgd)

{

	*pgdp = pgd;

	dsb(ishst);

}



static inline void pgd_clear(pgd_t *pgdp)

{

	set_pgd(pgdp, __pgd(0));

}



static inline pud_t *pgd_page_vaddr(pgd_t pgd)

{

	return __va(pgd_val(pgd) & PHYS_MASK & (s32)PAGE_MASK);

}



#endif  /* CONFIG_ARM64_4_LEVELS */



/* to find an entry in a page-table-directory */

#define pgd_index(addr)		(((addr) >> PGDIR_SHIFT) & (PTRS_PER_PGD - 1))
@@ -355,6 +387,14 @@ static inline pmd_t *pud_page_vaddr(pud_t pud) 
/* to find an entry in a kernel page-table-directory */

#define pgd_offset_k(addr)	pgd_offset(&init_mm, addr)



#ifdef CONFIG_ARM64_4_LEVELS

#define pud_index(addr)		(((addr) >> PUD_SHIFT) & (PTRS_PER_PUD - 1))

static inline pud_t *pud_offset(pgd_t *pgd, unsigned long addr)

{

	return (pud_t *)pgd_page_vaddr(*pgd) + pud_index(addr);

}

#endif



/* Find an entry in the second-level page table.. */

#ifndef CONFIG_ARM64_2_LEVELS

#define pmd_index(addr)		(((addr) >> PMD_SHIFT) & (PTRS_PER_PMD - 1))

CRA Git | Maintained and supported by SUSTech CRA and CCSE