Commit 90292aca authored by Yu Zhao's avatar Yu Zhao Committed by Will Deacon
Browse files

arm64: mm: use appropriate ctors for page tables 



For pte page, use pgtable_page_ctor(); for pmd page, use
pgtable_pmd_page_ctor(); and for the rest (pud, p4d and pgd),
don't use any.

For now, we don't select ARCH_ENABLE_SPLIT_PMD_PTLOCK and
pgtable_pmd_page_ctor() is a nop. When we do in patch 3, we
make sure pmd is not folded so we won't mistakenly call
pgtable_pmd_page_ctor() on pud or p4d.

Acked-by: default avatarMark Rutland <mark.rutland@arm.com>
Signed-off-by: default avatarYu Zhao <yuzhao@google.com>
Signed-off-by: default avatarWill Deacon <will.deacon@arm.com>
parent ab6211c9

arch/arm64/mm/mmu.c

+24 −12
Original line number Diff line number Diff line
@@ -97,7 +97,7 @@ pgprot_t phys_mem_access_prot(struct file *file, unsigned long pfn, 
}

EXPORT_SYMBOL(phys_mem_access_prot);



static phys_addr_t __init early_pgtable_alloc(void)

static phys_addr_t __init early_pgtable_alloc(int shift)

{

	phys_addr_t phys;

	void *ptr;
@@ -174,7 +174,7 @@ static void init_pte(pmd_t *pmdp, unsigned long addr, unsigned long end, 
static void alloc_init_cont_pte(pmd_t *pmdp, unsigned long addr,

				unsigned long end, phys_addr_t phys,

				pgprot_t prot,

				phys_addr_t (*pgtable_alloc)(void),

				phys_addr_t (*pgtable_alloc)(int),

				int flags)

{

	unsigned long next;
@@ -184,7 +184,7 @@ static void alloc_init_cont_pte(pmd_t *pmdp, unsigned long addr, 
	if (pmd_none(pmd)) {

		phys_addr_t pte_phys;

		BUG_ON(!pgtable_alloc);

		pte_phys = pgtable_alloc();

		pte_phys = pgtable_alloc(PAGE_SHIFT);

		__pmd_populate(pmdp, pte_phys, PMD_TYPE_TABLE);

		pmd = READ_ONCE(*pmdp);

	}
@@ -208,7 +208,7 @@ static void alloc_init_cont_pte(pmd_t *pmdp, unsigned long addr, 


static void init_pmd(pud_t *pudp, unsigned long addr, unsigned long end,

		     phys_addr_t phys, pgprot_t prot,

		     phys_addr_t (*pgtable_alloc)(void), int flags)

		     phys_addr_t (*pgtable_alloc)(int), int flags)

{

	unsigned long next;

	pmd_t *pmdp;
@@ -246,7 +246,7 @@ static void init_pmd(pud_t *pudp, unsigned long addr, unsigned long end, 
static void alloc_init_cont_pmd(pud_t *pudp, unsigned long addr,

				unsigned long end, phys_addr_t phys,

				pgprot_t prot,

				phys_addr_t (*pgtable_alloc)(void), int flags)

				phys_addr_t (*pgtable_alloc)(int), int flags)

{

	unsigned long next;

	pud_t pud = READ_ONCE(*pudp);
@@ -258,7 +258,7 @@ static void alloc_init_cont_pmd(pud_t *pudp, unsigned long addr, 
	if (pud_none(pud)) {

		phys_addr_t pmd_phys;

		BUG_ON(!pgtable_alloc);

		pmd_phys = pgtable_alloc();

		pmd_phys = pgtable_alloc(PMD_SHIFT);

		__pud_populate(pudp, pmd_phys, PUD_TYPE_TABLE);

		pud = READ_ONCE(*pudp);

	}
@@ -294,7 +294,7 @@ static inline bool use_1G_block(unsigned long addr, unsigned long next, 


static void alloc_init_pud(pgd_t *pgdp, unsigned long addr, unsigned long end,

			   phys_addr_t phys, pgprot_t prot,

			   phys_addr_t (*pgtable_alloc)(void),

			   phys_addr_t (*pgtable_alloc)(int),

			   int flags)

{

	unsigned long next;
@@ -304,7 +304,7 @@ static void alloc_init_pud(pgd_t *pgdp, unsigned long addr, unsigned long end, 
	if (pgd_none(pgd)) {

		phys_addr_t pud_phys;

		BUG_ON(!pgtable_alloc);

		pud_phys = pgtable_alloc();

		pud_phys = pgtable_alloc(PUD_SHIFT);

		__pgd_populate(pgdp, pud_phys, PUD_TYPE_TABLE);

		pgd = READ_ONCE(*pgdp);

	}
@@ -345,7 +345,7 @@ static void alloc_init_pud(pgd_t *pgdp, unsigned long addr, unsigned long end, 
static void __create_pgd_mapping(pgd_t *pgdir, phys_addr_t phys,

				 unsigned long virt, phys_addr_t size,

				 pgprot_t prot,

				 phys_addr_t (*pgtable_alloc)(void),

				 phys_addr_t (*pgtable_alloc)(int),

				 int flags)

{

	unsigned long addr, length, end, next;
@@ -371,11 +371,23 @@ static void __create_pgd_mapping(pgd_t *pgdir, phys_addr_t phys, 
	} while (pgdp++, addr = next, addr != end);

}



static phys_addr_t pgd_pgtable_alloc(void)

static phys_addr_t pgd_pgtable_alloc(int shift)

{

	void *ptr = (void *)__get_free_page(PGALLOC_GFP);

	if (!ptr || !pgtable_page_ctor(virt_to_page(ptr)))

		BUG();

	BUG_ON(!ptr);



	/*

	 * Call proper page table ctor in case later we need to

	 * call core mm functions like apply_to_page_range() on

	 * this pre-allocated page table.

	 *

	 * We don't select ARCH_ENABLE_SPLIT_PMD_PTLOCK if pmd is

	 * folded, and if so pgtable_pmd_page_ctor() becomes nop.

	 */

	if (shift == PAGE_SHIFT)

		BUG_ON(!pgtable_page_ctor(virt_to_page(ptr)));

	else if (shift == PMD_SHIFT)

		BUG_ON(!pgtable_pmd_page_ctor(virt_to_page(ptr)));



	/* Ensure the zeroed page is visible to the page table walker */

	dsb(ishst);

CRA Git | Maintained and supported by SUSTech CRA and CCSE