Commit 84e6ffb2 authored by Mike Rapoport's avatar Mike Rapoport Committed by Linus Torvalds
Browse files

arm: add support for folded p4d page tables 

Implement primitives necessary for the 4th level folding, add walks of p4d
level where appropriate, and remove __ARCH_USE_5LEVEL_HACK.

[rppt@linux.ibm.com: fix kexec]
  Link: http://lkml.kernel.org/r/20200508174232.GA759899@linux.ibm.com


Signed-off-by: default avatarMike Rapoport <rppt@linux.ibm.com>
Signed-off-by: default avatarAndrew Morton <akpm@linux-foundation.org>
Tested-by: default avatarMarek Szyprowski <m.szyprowski@samsung.com>
Cc: Arnd Bergmann <arnd@arndb.de>
Cc: Benjamin Herrenschmidt <benh@kernel.crashing.org>
Cc: Brian Cain <bcain@codeaurora.org>
Cc: Catalin Marinas <catalin.marinas@arm.com>
Cc: Christophe Leroy <christophe.leroy@c-s.fr>
Cc: Fenghua Yu <fenghua.yu@intel.com>
Cc: Geert Uytterhoeven <geert+renesas@glider.be>
Cc: Guan Xuetao <gxt@pku.edu.cn>
Cc: James Morse <james.morse@arm.com>
Cc: Jonas Bonn <jonas@southpole.se>
Cc: Julien Thierry <julien.thierry.kdev@gmail.com>
Cc: Ley Foon Tan <ley.foon.tan@intel.com>
Cc: Marc Zyngier <maz@kernel.org>
Cc: Michael Ellerman <mpe@ellerman.id.au>
Cc: Paul Mackerras <paulus@samba.org>
Cc: Rich Felker <dalias@libc.org>
Cc: Russell King <linux@armlinux.org.uk>
Cc: Stafford Horne <shorne@gmail.com>
Cc: Stefan Kristiansson <stefan.kristiansson@saunalahti.fi>
Cc: Suzuki K Poulose <suzuki.poulose@arm.com>
Cc: Tony Luck <tony.luck@intel.com>
Cc: Will Deacon <will@kernel.org>
Cc: Yoshinori Sato <ysato@users.sourceforge.jp>
Link: http://lkml.kernel.org/r/20200414153455.21744-3-rppt@kernel.org


Signed-off-by: default avatarLinus Torvalds <torvalds@linux-foundation.org>
parent f426f4ed

arch/arm/include/asm/pgtable.h

+0 −1
Original line number Diff line number Diff line
@@ -17,7 +17,6 @@ 


#else



#define __ARCH_USE_5LEVEL_HACK

#include <asm-generic/pgtable-nopud.h>

#include <asm/memory.h>

#include <asm/pgtable-hwdef.h>

arch/arm/lib/uaccess_with_memcpy.c

+6 −1
Original line number Diff line number Diff line
@@ -24,6 +24,7 @@ pin_page_for_write(const void __user *_addr, pte_t **ptep, spinlock_t **ptlp) 
{

	unsigned long addr = (unsigned long)_addr;

	pgd_t *pgd;

	p4d_t *p4d;

	pmd_t *pmd;

	pte_t *pte;

	pud_t *pud;
@@ -33,7 +34,11 @@ pin_page_for_write(const void __user *_addr, pte_t **ptep, spinlock_t **ptlp) 
	if (unlikely(pgd_none(*pgd) || pgd_bad(*pgd)))

		return 0;



	pud = pud_offset(pgd, addr);

	p4d = p4d_offset(pgd, addr);

	if (unlikely(p4d_none(*p4d) || p4d_bad(*p4d)))

		return 0;



	pud = pud_offset(p4d, addr);

	if (unlikely(pud_none(*pud) || pud_bad(*pud)))

		return 0;

arch/arm/mach-sa1100/assabet.c

+1 −1
Original line number Diff line number Diff line
@@ -633,7 +633,7 @@ static void __init map_sa1100_gpio_regs( void ) 
	int prot = PMD_TYPE_SECT | PMD_SECT_AP_WRITE | PMD_DOMAIN(DOMAIN_IO);

	pmd_t *pmd;



	pmd = pmd_offset(pud_offset(pgd_offset_k(virt), virt), virt);

	pmd = pmd_offset(pud_offset(p4d_offset(pgd_offset_k(virt), virt), virt), virt);

	*pmd = __pmd(phys | prot);

	flush_pmd_entry(pmd);

}

arch/arm/mm/dump.c

+23 −6
Original line number Diff line number Diff line
@@ -207,6 +207,7 @@ struct pg_level { 
static struct pg_level pg_level[] = {

	{

	}, { /* pgd */

	}, { /* p4d */

	}, { /* pud */

	}, { /* pmd */

		.bits	= section_bits,
@@ -308,7 +309,7 @@ static void walk_pte(struct pg_state *st, pmd_t *pmd, unsigned long start, 


	for (i = 0; i < PTRS_PER_PTE; i++, pte++) {

		addr = start + i * PAGE_SIZE;

		note_page(st, addr, 4, pte_val(*pte), domain);

		note_page(st, addr, 5, pte_val(*pte), domain);

	}

}
@@ -350,14 +351,14 @@ static void walk_pmd(struct pg_state *st, pud_t *pud, unsigned long start) 
			addr += SECTION_SIZE;

			pmd++;

			domain = get_domain_name(pmd);

			note_page(st, addr, 3, pmd_val(*pmd), domain);

			note_page(st, addr, 4, pmd_val(*pmd), domain);

		}

	}

}



static void walk_pud(struct pg_state *st, pgd_t *pgd, unsigned long start)

static void walk_pud(struct pg_state *st, p4d_t *p4d, unsigned long start)

{

	pud_t *pud = pud_offset(pgd, 0);

	pud_t *pud = pud_offset(p4d, 0);

	unsigned long addr;

	unsigned i;
@@ -366,7 +367,23 @@ static void walk_pud(struct pg_state *st, pgd_t *pgd, unsigned long start) 
		if (!pud_none(*pud)) {

			walk_pmd(st, pud, addr);

		} else {

			note_page(st, addr, 2, pud_val(*pud), NULL);

			note_page(st, addr, 3, pud_val(*pud), NULL);

		}

	}

}



static void walk_p4d(struct pg_state *st, pgd_t *pgd, unsigned long start)

{

	p4d_t *p4d = p4d_offset(pgd, 0);

	unsigned long addr;

	unsigned i;



	for (i = 0; i < PTRS_PER_P4D; i++, p4d++) {

		addr = start + i * P4D_SIZE;

		if (!p4d_none(*p4d)) {

			walk_pud(st, p4d, addr);

		} else {

			note_page(st, addr, 2, p4d_val(*p4d), NULL);

		}

	}

}
@@ -381,7 +398,7 @@ static void walk_pgd(struct pg_state *st, struct mm_struct *mm, 
	for (i = 0; i < PTRS_PER_PGD; i++, pgd++) {

		addr = start + i * PGDIR_SIZE;

		if (!pgd_none(*pgd)) {

			walk_pud(st, pgd, addr);

			walk_p4d(st, pgd, addr);

		} else {

			note_page(st, addr, 1, pgd_val(*pgd), NULL);

		}

arch/arm/mm/fault-armv.c

+6 −1
Original line number Diff line number Diff line
@@ -91,6 +91,7 @@ static int adjust_pte(struct vm_area_struct *vma, unsigned long address, 
{

	spinlock_t *ptl;

	pgd_t *pgd;

	p4d_t *p4d;

	pud_t *pud;

	pmd_t *pmd;

	pte_t *pte;
@@ -100,7 +101,11 @@ static int adjust_pte(struct vm_area_struct *vma, unsigned long address, 
	if (pgd_none_or_clear_bad(pgd))

		return 0;



	pud = pud_offset(pgd, address);

	p4d = p4d_offset(pgd, address);

	if (p4d_none_or_clear_bad(p4d))

		return 0;



	pud = pud_offset(p4d, address);

	if (pud_none_or_clear_bad(pud))

		return 0;

CRA Git | Maintained and supported by SUSTech CRA and CCSE