Commit 75387b92 authored by Kristina Martsenko's avatar Kristina Martsenko Committed by Catalin Marinas
Browse files

arm64: handle 52-bit physical addresses in page table entries 



The top 4 bits of a 52-bit physical address are positioned at bits
12..15 of a page table entry. Introduce macros to convert between a
physical address and its placement in a table entry, and change all
macros/functions that access PTEs to use them.

Reviewed-by: default avatarMarc Zyngier <marc.zyngier@arm.com>
Tested-by: default avatarSuzuki K Poulose <suzuki.poulose@arm.com>
Reviewed-by: default avatarSuzuki K Poulose <suzuki.poulose@arm.com>
Tested-by: default avatarBob Picco <bob.picco@oracle.com>
Reviewed-by: default avatarBob Picco <bob.picco@oracle.com>
Signed-off-by: default avatarKristina Martsenko <kristina.martsenko@arm.com>
[catalin.marinas@arm.com: some long lines wrapped]
Signed-off-by: default avatarCatalin Marinas <catalin.marinas@arm.com>
parent 19338304

arch/arm64/include/asm/kvm_mmu.h

+5 −2
Original line number Diff line number Diff line
@@ -287,6 +287,7 @@ static inline void __kvm_extend_hypmap(pgd_t *boot_hyp_pgd, 
				       unsigned long hyp_idmap_start)

{

	int idmap_idx;

	u64 pgd_addr;



	/*

	 * Use the first entry to access the HYP mappings. It is
@@ -294,7 +295,8 @@ static inline void __kvm_extend_hypmap(pgd_t *boot_hyp_pgd, 
	 * extended idmap.

	 */

	VM_BUG_ON(pgd_val(merged_hyp_pgd[0]));

	merged_hyp_pgd[0] = __pgd(__pa(hyp_pgd) | PMD_TYPE_TABLE);

	pgd_addr = __phys_to_pgd_val(__pa(hyp_pgd));

	merged_hyp_pgd[0] = __pgd(pgd_addr | PMD_TYPE_TABLE);



	/*

	 * Create another extended level entry that points to the boot HYP map,
@@ -304,7 +306,8 @@ static inline void __kvm_extend_hypmap(pgd_t *boot_hyp_pgd, 
	 */

	idmap_idx = hyp_idmap_start >> VA_BITS;

	VM_BUG_ON(pgd_val(merged_hyp_pgd[idmap_idx]));

	merged_hyp_pgd[idmap_idx] = __pgd(__pa(boot_hyp_pgd) | PMD_TYPE_TABLE);

	pgd_addr = __phys_to_pgd_val(__pa(boot_hyp_pgd));

	merged_hyp_pgd[idmap_idx] = __pgd(pgd_addr | PMD_TYPE_TABLE);

}



static inline unsigned int kvm_get_vmid_bits(void)

arch/arm64/include/asm/pgalloc.h

+3 −3
Original line number Diff line number Diff line
@@ -44,7 +44,7 @@ static inline void pmd_free(struct mm_struct *mm, pmd_t *pmd) 


static inline void __pud_populate(pud_t *pud, phys_addr_t pmd, pudval_t prot)

{

	set_pud(pud, __pud(pmd | prot));

	set_pud(pud, __pud(__phys_to_pud_val(pmd) | prot));

}



static inline void pud_populate(struct mm_struct *mm, pud_t *pud, pmd_t *pmd)
@@ -73,7 +73,7 @@ static inline void pud_free(struct mm_struct *mm, pud_t *pud) 


static inline void __pgd_populate(pgd_t *pgdp, phys_addr_t pud, pgdval_t prot)

{

	set_pgd(pgdp, __pgd(pud | prot));

	set_pgd(pgdp, __pgd(__phys_to_pgd_val(pud) | prot));

}



static inline void pgd_populate(struct mm_struct *mm, pgd_t *pgd, pud_t *pud)
@@ -129,7 +129,7 @@ static inline void pte_free(struct mm_struct *mm, pgtable_t pte) 
static inline void __pmd_populate(pmd_t *pmdp, phys_addr_t pte,

				  pmdval_t prot)

{

	set_pmd(pmdp, __pmd(pte | prot));

	set_pmd(pmdp, __pmd(__phys_to_pmd_val(pte) | prot));

}



/*

arch/arm64/include/asm/pgtable-hwdef.h

+4 −2
Original line number Diff line number Diff line
@@ -168,10 +168,12 @@ 
#define PTE_UXN			(_AT(pteval_t, 1) << 54)	/* User XN */

#define PTE_HYP_XN		(_AT(pteval_t, 1) << 54)	/* HYP XN */



#ifdef CONFIG_ARM64_PA_BITS_52

#define PTE_ADDR_LOW		(((_AT(pteval_t, 1) << (48 - PAGE_SHIFT)) - 1) << PAGE_SHIFT)

#ifdef CONFIG_ARM64_PA_BITS_52

#define PTE_ADDR_HIGH		(_AT(pteval_t, 0xf) << 12)

#define PTE_ADDR_MASK_52	(PTE_ADDR_LOW | PTE_ADDR_HIGH)

#define PTE_ADDR_MASK		(PTE_ADDR_LOW | PTE_ADDR_HIGH)

#else

#define PTE_ADDR_MASK		PTE_ADDR_LOW

#endif



/*

arch/arm64/include/asm/pgtable.h

+38 −12
Original line number Diff line number Diff line
@@ -57,9 +57,22 @@ extern unsigned long empty_zero_page[PAGE_SIZE / sizeof(unsigned long)]; 


#define pte_ERROR(pte)		__pte_error(__FILE__, __LINE__, pte_val(pte))



#define pte_pfn(pte)		((pte_val(pte) & PHYS_MASK) >> PAGE_SHIFT)

/*

 * Macros to convert between a physical address and its placement in a

 * page table entry, taking care of 52-bit addresses.

 */

#ifdef CONFIG_ARM64_PA_BITS_52

#define __pte_to_phys(pte)	\

	((pte_val(pte) & PTE_ADDR_LOW) | ((pte_val(pte) & PTE_ADDR_HIGH) << 36))

#define __phys_to_pte_val(phys)	(((phys) | ((phys) >> 36)) & PTE_ADDR_MASK)

#else

#define __pte_to_phys(pte)	(pte_val(pte) & PTE_ADDR_MASK)

#define __phys_to_pte_val(phys)	(phys)

#endif



#define pfn_pte(pfn,prot)	(__pte(((phys_addr_t)(pfn) << PAGE_SHIFT) | pgprot_val(prot)))

#define pte_pfn(pte)		(__pte_to_phys(pte) >> PAGE_SHIFT)

#define pfn_pte(pfn,prot)	\

	__pte(__phys_to_pte_val((phys_addr_t)(pfn) << PAGE_SHIFT) | pgprot_val(prot))



#define pte_none(pte)		(!pte_val(pte))

#define pte_clear(mm,addr,ptep)	set_pte(ptep, __pte(0))
@@ -284,6 +297,11 @@ static inline int pte_same(pte_t pte_a, pte_t pte_b) 


#define __HAVE_ARCH_PTE_SPECIAL



static inline pte_t pgd_pte(pgd_t pgd)

{

	return __pte(pgd_val(pgd));

}



static inline pte_t pud_pte(pud_t pud)

{

	return __pte(pud_val(pud));
@@ -349,16 +367,24 @@ static inline int pmd_protnone(pmd_t pmd) 


#define pmd_mkhuge(pmd)		(__pmd(pmd_val(pmd) & ~PMD_TABLE_BIT))



#define pmd_pfn(pmd)		(((pmd_val(pmd) & PMD_MASK) & PHYS_MASK) >> PAGE_SHIFT)

#define pfn_pmd(pfn,prot)	(__pmd(((phys_addr_t)(pfn) << PAGE_SHIFT) | pgprot_val(prot)))

#define __pmd_to_phys(pmd)	__pte_to_phys(pmd_pte(pmd))

#define __phys_to_pmd_val(phys)	__phys_to_pte_val(phys)

#define pmd_pfn(pmd)		((__pmd_to_phys(pmd) & PMD_MASK) >> PAGE_SHIFT)

#define pfn_pmd(pfn,prot)	__pmd(__phys_to_pmd_val((phys_addr_t)(pfn) << PAGE_SHIFT) | pgprot_val(prot))

#define mk_pmd(page,prot)	pfn_pmd(page_to_pfn(page),prot)



#define pud_write(pud)		pte_write(pud_pte(pud))

#define pud_pfn(pud)		(((pud_val(pud) & PUD_MASK) & PHYS_MASK) >> PAGE_SHIFT)

#define pfn_pud(pfn,prot)	(__pud(((phys_addr_t)(pfn) << PAGE_SHIFT) | pgprot_val(prot)))



#define __pud_to_phys(pud)	__pte_to_phys(pud_pte(pud))

#define __phys_to_pud_val(phys)	__phys_to_pte_val(phys)

#define pud_pfn(pud)		((__pud_to_phys(pud) & PUD_MASK) >> PAGE_SHIFT)

#define pfn_pud(pfn,prot)	__pud(__phys_to_pud_val((phys_addr_t)(pfn) << PAGE_SHIFT) | pgprot_val(prot))



#define set_pmd_at(mm, addr, pmdp, pmd)	set_pte_at(mm, addr, (pte_t *)pmdp, pmd_pte(pmd))



#define __pgd_to_phys(pgd)	__pte_to_phys(pgd_pte(pgd))

#define __phys_to_pgd_val(phys)	__phys_to_pte_val(phys)



#define __pgprot_modify(prot,mask,bits) \

	__pgprot((pgprot_val(prot) & ~(mask)) | (bits))
@@ -409,7 +435,7 @@ static inline void pmd_clear(pmd_t *pmdp) 


static inline phys_addr_t pmd_page_paddr(pmd_t pmd)

{

	return pmd_val(pmd) & PHYS_MASK & (s32)PAGE_MASK;

	return __pmd_to_phys(pmd);

}



/* Find an entry in the third-level page table. */
@@ -427,7 +453,7 @@ static inline phys_addr_t pmd_page_paddr(pmd_t pmd) 
#define pte_set_fixmap_offset(pmd, addr)	pte_set_fixmap(pte_offset_phys(pmd, addr))

#define pte_clear_fixmap()		clear_fixmap(FIX_PTE)



#define pmd_page(pmd)		pfn_to_page(__phys_to_pfn(pmd_val(pmd) & PHYS_MASK))

#define pmd_page(pmd)		pfn_to_page(__phys_to_pfn(__pmd_to_phys(pmd)))



/* use ONLY for statically allocated translation tables */

#define pte_offset_kimg(dir,addr)	((pte_t *)__phys_to_kimg(pte_offset_phys((dir), (addr))))
@@ -460,7 +486,7 @@ static inline void pud_clear(pud_t *pudp) 


static inline phys_addr_t pud_page_paddr(pud_t pud)

{

	return pud_val(pud) & PHYS_MASK & (s32)PAGE_MASK;

	return __pud_to_phys(pud);

}



/* Find an entry in the second-level page table. */
@@ -473,7 +499,7 @@ static inline phys_addr_t pud_page_paddr(pud_t pud) 
#define pmd_set_fixmap_offset(pud, addr)	pmd_set_fixmap(pmd_offset_phys(pud, addr))

#define pmd_clear_fixmap()		clear_fixmap(FIX_PMD)



#define pud_page(pud)		pfn_to_page(__phys_to_pfn(pud_val(pud) & PHYS_MASK))

#define pud_page(pud)		pfn_to_page(__phys_to_pfn(__pud_to_phys(pud)))



/* use ONLY for statically allocated translation tables */

#define pmd_offset_kimg(dir,addr)	((pmd_t *)__phys_to_kimg(pmd_offset_phys((dir), (addr))))
@@ -512,7 +538,7 @@ static inline void pgd_clear(pgd_t *pgdp) 


static inline phys_addr_t pgd_page_paddr(pgd_t pgd)

{

	return pgd_val(pgd) & PHYS_MASK & (s32)PAGE_MASK;

	return __pgd_to_phys(pgd);

}



/* Find an entry in the frst-level page table. */
@@ -525,7 +551,7 @@ static inline phys_addr_t pgd_page_paddr(pgd_t pgd) 
#define pud_set_fixmap_offset(pgd, addr)	pud_set_fixmap(pud_offset_phys(pgd, addr))

#define pud_clear_fixmap()		clear_fixmap(FIX_PUD)



#define pgd_page(pgd)		pfn_to_page(__phys_to_pfn(pgd_val(pgd) & PHYS_MASK))

#define pgd_page(pgd)		pfn_to_page(__phys_to_pfn(__pgd_to_phys(pgd)))



/* use ONLY for statically allocated translation tables */

#define pud_offset_kimg(dir,addr)	((pud_t *)__phys_to_kimg(pud_offset_phys((dir), (addr))))

arch/arm64/kernel/head.S

+1 −1
Original line number Diff line number Diff line
@@ -161,7 +161,7 @@ ENDPROC(preserve_boot_args) 
	 * supporting this configuration with 64K pages.

	 */

	orr	\pte, \phys, \phys, lsr #36

	and	\pte, \pte, #PTE_ADDR_MASK_52

	and	\pte, \pte, #PTE_ADDR_MASK

#else

	mov	\pte, \phys

#endif

CRA Git | Maintained and supported by SUSTech CRA and CCSE