Commit 8d962507 authored by Catalin Marinas's avatar Catalin Marinas Committed by Steve Capper
Browse files

ARM: mm: Transparent huge page support for LPAE systems. 



The patch adds support for THP (transparent huge pages) to LPAE
systems. When this feature is enabled, the kernel tries to map
anonymous pages as 2MB sections where possible.

Signed-off-by: default avatarCatalin Marinas <catalin.marinas@arm.com>
[steve.capper@linaro.org: symbolic constants used, value of
PMD_SECT_SPLITTING adjusted, tlbflush.h included in pgtable.h,
added PROT_NONE support.]
Signed-off-by: default avatarSteve Capper <steve.capper@linaro.org>
Reviewed-by: default avatarWill Deacon <will.deacon@arm.com>
parent 1355e2a6

arch/arm/Kconfig

+4 −0
Original line number Diff line number Diff line
@@ -1711,6 +1711,10 @@ config SYS_SUPPORTS_HUGETLBFS 
       def_bool y

       depends on ARM_LPAE



config HAVE_ARCH_TRANSPARENT_HUGEPAGE

       def_bool y

       depends on ARM_LPAE



source "mm/Kconfig"



config FORCE_MAX_ZONEORDER

arch/arm/include/asm/pgtable-3level-hwdef.h

+2 −0
Original line number Diff line number Diff line
@@ -42,6 +42,8 @@ 
 */

#define PMD_SECT_BUFFERABLE	(_AT(pmdval_t, 1) << 2)

#define PMD_SECT_CACHEABLE	(_AT(pmdval_t, 1) << 3)

#define PMD_SECT_USER		(_AT(pmdval_t, 1) << 6)		/* AP[1] */

#define PMD_SECT_RDONLY		(_AT(pmdval_t, 1) << 7)		/* AP[2] */

#define PMD_SECT_S		(_AT(pmdval_t, 3) << 8)

#define PMD_SECT_AF		(_AT(pmdval_t, 1) << 10)

#define PMD_SECT_nG		(_AT(pmdval_t, 1) << 11)

arch/arm/include/asm/pgtable-3level.h

+60 −0
Original line number Diff line number Diff line
@@ -87,6 +87,11 @@ 
#define L_PTE_SPECIAL		(_AT(pteval_t, 1) << 56)	/* unused */

#define L_PTE_NONE		(_AT(pteval_t, 1) << 57)	/* PROT_NONE */



#define PMD_SECT_VALID		(_AT(pmdval_t, 1) << 0)

#define PMD_SECT_DIRTY		(_AT(pmdval_t, 1) << 55)

#define PMD_SECT_SPLITTING	(_AT(pmdval_t, 1) << 56)

#define PMD_SECT_NONE		(_AT(pmdval_t, 1) << 57)



/*

 * To be used in assembly code with the upper page attributes.

 */
@@ -196,6 +201,61 @@ static inline pmd_t *pmd_offset(pud_t *pud, unsigned long addr) 
#define pte_huge(pte)		(pte_val(pte) && !(pte_val(pte) & PTE_TABLE_BIT))

#define pte_mkhuge(pte)		(__pte(pte_val(pte) & ~PTE_TABLE_BIT))



#define pmd_young(pmd)		(pmd_val(pmd) & PMD_SECT_AF)



#define __HAVE_ARCH_PMD_WRITE

#define pmd_write(pmd)		(!(pmd_val(pmd) & PMD_SECT_RDONLY))



#ifdef CONFIG_TRANSPARENT_HUGEPAGE

#define pmd_trans_huge(pmd)	(pmd_val(pmd) && !(pmd_val(pmd) & PMD_TABLE_BIT))

#define pmd_trans_splitting(pmd) (pmd_val(pmd) & PMD_SECT_SPLITTING)

#endif



#define PMD_BIT_FUNC(fn,op) \

static inline pmd_t pmd_##fn(pmd_t pmd) { pmd_val(pmd) op; return pmd; }



PMD_BIT_FUNC(wrprotect,	|= PMD_SECT_RDONLY);

PMD_BIT_FUNC(mkold,	&= ~PMD_SECT_AF);

PMD_BIT_FUNC(mksplitting, |= PMD_SECT_SPLITTING);

PMD_BIT_FUNC(mkwrite,   &= ~PMD_SECT_RDONLY);

PMD_BIT_FUNC(mkdirty,   |= PMD_SECT_DIRTY);

PMD_BIT_FUNC(mkyoung,   |= PMD_SECT_AF);



#define pmd_mkhuge(pmd)		(__pmd(pmd_val(pmd) & ~PMD_TABLE_BIT))



#define pmd_pfn(pmd)		(((pmd_val(pmd) & PMD_MASK) & PHYS_MASK) >> PAGE_SHIFT)

#define pfn_pmd(pfn,prot)	(__pmd(((phys_addr_t)(pfn) << PAGE_SHIFT) | pgprot_val(prot)))

#define mk_pmd(page,prot)	pfn_pmd(page_to_pfn(page),prot)



/* represent a notpresent pmd by zero, this is used by pmdp_invalidate */

#define pmd_mknotpresent(pmd)	(__pmd(0))



static inline pmd_t pmd_modify(pmd_t pmd, pgprot_t newprot)

{

	const pmdval_t mask = PMD_SECT_USER | PMD_SECT_XN | PMD_SECT_RDONLY |

				PMD_SECT_VALID | PMD_SECT_NONE;

	pmd_val(pmd) = (pmd_val(pmd) & ~mask) | (pgprot_val(newprot) & mask);

	return pmd;

}



static inline void set_pmd_at(struct mm_struct *mm, unsigned long addr,

			      pmd_t *pmdp, pmd_t pmd)

{

	BUG_ON(addr >= TASK_SIZE);



	/* create a faulting entry if PROT_NONE protected */

	if (pmd_val(pmd) & PMD_SECT_NONE)

		pmd_val(pmd) &= ~PMD_SECT_VALID;



	*pmdp = __pmd(pmd_val(pmd) | PMD_SECT_nG);

	flush_pmd_entry(pmdp);

}



static inline int has_transparent_hugepage(void)

{

	return 1;

}



#endif /* __ASSEMBLY__ */



#endif /* _ASM_PGTABLE_3LEVEL_H */

arch/arm/include/asm/pgtable.h

+3 −0
Original line number Diff line number Diff line
@@ -24,6 +24,9 @@ 
#include <asm/memory.h>

#include <asm/pgtable-hwdef.h>





#include <asm/tlbflush.h>



#ifdef CONFIG_ARM_LPAE

#include <asm/pgtable-3level.h>

#else

arch/arm/include/asm/tlb.h

+6 −0
Original line number Diff line number Diff line
@@ -223,6 +223,12 @@ static inline void __pmd_free_tlb(struct mmu_gather *tlb, pmd_t *pmdp, 
#endif

}



static inline void

tlb_remove_pmd_tlb_entry(struct mmu_gather *tlb, pmd_t *pmdp, unsigned long addr)

{

	tlb_add_flush(tlb, addr);

}



#define pte_free_tlb(tlb, ptep, addr)	__pte_free_tlb(tlb, ptep, addr)

#define pmd_free_tlb(tlb, pmdp, addr)	__pmd_free_tlb(tlb, pmdp, addr)

#define pud_free_tlb(tlb, pudp, addr)	pud_free((tlb)->mm, pudp)

CRA Git | Maintained and supported by SUSTech CRA and CCSE