Commit 2fb47060 authored by Mike Rapoport's avatar Mike Rapoport Committed by Linus Torvalds
Browse files

powerpc: add support for folded p4d page tables 

Implement primitives necessary for the 4th level folding, add walks of p4d
level where appropriate and replace 5level-fixup.h with pgtable-nop4d.h.

[rppt@linux.ibm.com: powerpc/xmon: drop unused pgdir varialble in show_pte() function]
  Link: http://lkml.kernel.org/r/20200519181454.GI1059226@linux.ibm.com
[rppt@linux.ibm.com; build fix]
  Link: http://lkml.kernel.org/r/20200423141845.GI13521@linux.ibm.com


Signed-off-by: default avatarMike Rapoport <rppt@linux.ibm.com>
Signed-off-by: default avatarAndrew Morton <akpm@linux-foundation.org>
Tested-by: Christophe Leroy <christophe.leroy@c-s.fr> # 8xx and 83xx
Cc: Arnd Bergmann <arnd@arndb.de>
Cc: Benjamin Herrenschmidt <benh@kernel.crashing.org>
Cc: Brian Cain <bcain@codeaurora.org>
Cc: Catalin Marinas <catalin.marinas@arm.com>
Cc: Fenghua Yu <fenghua.yu@intel.com>
Cc: Geert Uytterhoeven <geert+renesas@glider.be>
Cc: Guan Xuetao <gxt@pku.edu.cn>
Cc: James Morse <james.morse@arm.com>
Cc: Jonas Bonn <jonas@southpole.se>
Cc: Julien Thierry <julien.thierry.kdev@gmail.com>
Cc: Ley Foon Tan <ley.foon.tan@intel.com>
Cc: Marc Zyngier <maz@kernel.org>
Cc: Michael Ellerman <mpe@ellerman.id.au>
Cc: Paul Mackerras <paulus@samba.org>
Cc: Rich Felker <dalias@libc.org>
Cc: Russell King <linux@armlinux.org.uk>
Cc: Stafford Horne <shorne@gmail.com>
Cc: Stefan Kristiansson <stefan.kristiansson@saunalahti.fi>
Cc: Suzuki K Poulose <suzuki.poulose@arm.com>
Cc: Tony Luck <tony.luck@intel.com>
Cc: Will Deacon <will@kernel.org>
Cc: Yoshinori Sato <ysato@users.sourceforge.jp>
Link: http://lkml.kernel.org/r/20200414153455.21744-9-rppt@kernel.org


Signed-off-by: default avatarLinus Torvalds <torvalds@linux-foundation.org>
parent b187fb7f

arch/powerpc/include/asm/book3s/32/pgtable.h

+0 −1
Original line number Diff line number Diff line
@@ -2,7 +2,6 @@ 
#ifndef _ASM_POWERPC_BOOK3S_32_PGTABLE_H

#define _ASM_POWERPC_BOOK3S_32_PGTABLE_H



#define __ARCH_USE_5LEVEL_HACK

#include <asm-generic/pgtable-nopmd.h>



#include <asm/book3s/32/hash.h>

arch/powerpc/include/asm/book3s/64/hash.h

+2 −2
Original line number Diff line number Diff line
@@ -134,9 +134,9 @@ static inline int get_region_id(unsigned long ea) 


#define	hash__pmd_bad(pmd)		(pmd_val(pmd) & H_PMD_BAD_BITS)

#define	hash__pud_bad(pud)		(pud_val(pud) & H_PUD_BAD_BITS)

static inline int hash__pgd_bad(pgd_t pgd)

static inline int hash__p4d_bad(p4d_t p4d)

{

	return (pgd_val(pgd) == 0);

	return (p4d_val(p4d) == 0);

}

#ifdef CONFIG_STRICT_KERNEL_RWX

extern void hash__mark_rodata_ro(void);

arch/powerpc/include/asm/book3s/64/pgalloc.h

+2 −2
Original line number Diff line number Diff line
@@ -85,9 +85,9 @@ static inline void pgd_free(struct mm_struct *mm, pgd_t *pgd) 
	kmem_cache_free(PGT_CACHE(PGD_INDEX_SIZE), pgd);

}



static inline void pgd_populate(struct mm_struct *mm, pgd_t *pgd, pud_t *pud)

static inline void p4d_populate(struct mm_struct *mm, p4d_t *pgd, pud_t *pud)

{

	*pgd =  __pgd(__pgtable_ptr_val(pud) | PGD_VAL_BITS);

	*pgd =  __p4d(__pgtable_ptr_val(pud) | PGD_VAL_BITS);

}



static inline pud_t *pud_alloc_one(struct mm_struct *mm, unsigned long addr)

arch/powerpc/include/asm/book3s/64/pgtable.h

+33 −27
Original line number Diff line number Diff line
@@ -2,7 +2,7 @@ 
#ifndef _ASM_POWERPC_BOOK3S_64_PGTABLE_H_

#define _ASM_POWERPC_BOOK3S_64_PGTABLE_H_



#include <asm-generic/5level-fixup.h>

#include <asm-generic/pgtable-nop4d.h>



#ifndef __ASSEMBLY__

#include <linux/mmdebug.h>
@@ -251,7 +251,7 @@ extern unsigned long __pmd_frag_size_shift; 
/* Bits to mask out from a PUD to get to the PMD page */

#define PUD_MASKED_BITS		0xc0000000000000ffUL

/* Bits to mask out from a PGD to get to the PUD page */

#define PGD_MASKED_BITS		0xc0000000000000ffUL

#define P4D_MASKED_BITS		0xc0000000000000ffUL



/*

 * Used as an indicator for rcu callback functions
@@ -949,54 +949,60 @@ static inline bool pud_access_permitted(pud_t pud, bool write) 
	return pte_access_permitted(pud_pte(pud), write);

}



#define pgd_write(pgd)		pte_write(pgd_pte(pgd))

#define __p4d_raw(x)	((p4d_t) { __pgd_raw(x) })

static inline __be64 p4d_raw(p4d_t x)

{

	return pgd_raw(x.pgd);

}



#define p4d_write(p4d)		pte_write(p4d_pte(p4d))



static inline void pgd_clear(pgd_t *pgdp)

static inline void p4d_clear(p4d_t *p4dp)

{

	*pgdp = __pgd(0);

	*p4dp = __p4d(0);

}



static inline int pgd_none(pgd_t pgd)

static inline int p4d_none(p4d_t p4d)

{

	return !pgd_raw(pgd);

	return !p4d_raw(p4d);

}



static inline int pgd_present(pgd_t pgd)

static inline int p4d_present(p4d_t p4d)

{

	return !!(pgd_raw(pgd) & cpu_to_be64(_PAGE_PRESENT));

	return !!(p4d_raw(p4d) & cpu_to_be64(_PAGE_PRESENT));

}



static inline pte_t pgd_pte(pgd_t pgd)

static inline pte_t p4d_pte(p4d_t p4d)

{

	return __pte_raw(pgd_raw(pgd));

	return __pte_raw(p4d_raw(p4d));

}



static inline pgd_t pte_pgd(pte_t pte)

static inline p4d_t pte_p4d(pte_t pte)

{

	return __pgd_raw(pte_raw(pte));

	return __p4d_raw(pte_raw(pte));

}



static inline int pgd_bad(pgd_t pgd)

static inline int p4d_bad(p4d_t p4d)

{

	if (radix_enabled())

		return radix__pgd_bad(pgd);

	return hash__pgd_bad(pgd);

		return radix__p4d_bad(p4d);

	return hash__p4d_bad(p4d);

}



#define pgd_access_permitted pgd_access_permitted

static inline bool pgd_access_permitted(pgd_t pgd, bool write)

#define p4d_access_permitted p4d_access_permitted

static inline bool p4d_access_permitted(p4d_t p4d, bool write)

{

	return pte_access_permitted(pgd_pte(pgd), write);

	return pte_access_permitted(p4d_pte(p4d), write);

}



extern struct page *pgd_page(pgd_t pgd);

extern struct page *p4d_page(p4d_t p4d);



/* Pointers in the page table tree are physical addresses */

#define __pgtable_ptr_val(ptr)	__pa(ptr)



#define pmd_page_vaddr(pmd)	__va(pmd_val(pmd) & ~PMD_MASKED_BITS)

#define pud_page_vaddr(pud)	__va(pud_val(pud) & ~PUD_MASKED_BITS)

#define pgd_page_vaddr(pgd)	__va(pgd_val(pgd) & ~PGD_MASKED_BITS)

#define p4d_page_vaddr(p4d)	__va(p4d_val(p4d) & ~P4D_MASKED_BITS)



#define pgd_index(address) (((address) >> (PGDIR_SHIFT)) & (PTRS_PER_PGD - 1))

#define pud_index(address) (((address) >> (PUD_SHIFT)) & (PTRS_PER_PUD - 1))
@@ -1010,8 +1016,8 @@ extern struct page *pgd_page(pgd_t pgd); 


#define pgd_offset(mm, address)	 ((mm)->pgd + pgd_index(address))



#define pud_offset(pgdp, addr)	\

	(((pud_t *) pgd_page_vaddr(*(pgdp))) + pud_index(addr))

#define pud_offset(p4dp, addr)	\

	(((pud_t *) p4d_page_vaddr(*(p4dp))) + pud_index(addr))

#define pmd_offset(pudp,addr) \

	(((pmd_t *) pud_page_vaddr(*(pudp))) + pmd_index(addr))

#define pte_offset_kernel(dir,addr) \
@@ -1366,11 +1372,11 @@ static inline bool pud_is_leaf(pud_t pud) 
	return !!(pud_raw(pud) & cpu_to_be64(_PAGE_PTE));

}



#define pgd_is_leaf pgd_is_leaf

#define pgd_leaf pgd_is_leaf

static inline bool pgd_is_leaf(pgd_t pgd)

#define p4d_is_leaf p4d_is_leaf

#define p4d_leaf p4d_is_leaf

static inline bool p4d_is_leaf(p4d_t p4d)

{

	return !!(pgd_raw(pgd) & cpu_to_be64(_PAGE_PTE));

	return !!(p4d_raw(p4d) & cpu_to_be64(_PAGE_PTE));

}



#endif /* __ASSEMBLY__ */

arch/powerpc/include/asm/book3s/64/radix.h

+3 −3
Original line number Diff line number Diff line
@@ -30,7 +30,7 @@ 
/* Don't have anything in the reserved bits and leaf bits */

#define RADIX_PMD_BAD_BITS		0x60000000000000e0UL

#define RADIX_PUD_BAD_BITS		0x60000000000000e0UL

#define RADIX_PGD_BAD_BITS		0x60000000000000e0UL

#define RADIX_P4D_BAD_BITS		0x60000000000000e0UL



#define RADIX_PMD_SHIFT		(PAGE_SHIFT + RADIX_PTE_INDEX_SIZE)

#define RADIX_PUD_SHIFT		(RADIX_PMD_SHIFT + RADIX_PMD_INDEX_SIZE)
@@ -227,9 +227,9 @@ static inline int radix__pud_bad(pud_t pud) 
}





static inline int radix__pgd_bad(pgd_t pgd)

static inline int radix__p4d_bad(p4d_t p4d)

{

	return !!(pgd_val(pgd) & RADIX_PGD_BAD_BITS);

	return !!(p4d_val(p4d) & RADIX_P4D_BAD_BITS);

}



#ifdef CONFIG_TRANSPARENT_HUGEPAGE

CRA Git | Maintained and supported by SUSTech CRA and CCSE