Commit 2ce77f6d authored by Will Deacon's avatar Will Deacon Committed by Catalin Marinas
Browse files

arm64: proc: Set PTE_NG for table entries to avoid traversing them twice 



When KASAN is enabled, the swapper page table contains many identical
mappings of the zero page, which can lead to a stall during boot whilst
the G -> nG code continually walks the same page table entries looking
for global mappings.

This patch sets the nG bit (bit 11, which is IGNORED) in table entries
after processing the subtree so we can easily skip them if we see them
a second time.

Tested-by: default avatarMark Rutland <mark.rutland@arm.com>
Signed-off-by: default avatarWill Deacon <will.deacon@arm.com>
Signed-off-by: default avatarCatalin Marinas <catalin.marinas@arm.com>
parent 16e574d7

arch/arm64/mm/proc.S

+9 −5
Original line number Diff line number Diff line
@@ -205,7 +205,8 @@ ENDPROC(idmap_cpu_replace_ttbr1) 
	dc	cvac, cur_\()\type\()p		// Ensure any existing dirty

	dmb	sy				// lines are written back before

	ldr	\type, [cur_\()\type\()p]	// loading the entry

	tbz	\type, #0, next_\()\type	// Skip invalid entries

	tbz	\type, #0, skip_\()\type	// Skip invalid and

	tbnz	\type, #11, skip_\()\type	// non-global entries

	.endm



	.macro __idmap_kpti_put_pgtable_ent_ng, type
@@ -265,8 +266,9 @@ ENTRY(idmap_kpti_install_ng_mappings) 
	add	end_pgdp, cur_pgdp, #(PTRS_PER_PGD * 8)

do_pgd:	__idmap_kpti_get_pgtable_ent	pgd

	tbnz	pgd, #1, walk_puds

	__idmap_kpti_put_pgtable_ent_ng	pgd

next_pgd:

	__idmap_kpti_put_pgtable_ent_ng	pgd

skip_pgd:

	add	cur_pgdp, cur_pgdp, #8

	cmp	cur_pgdp, end_pgdp

	b.ne	do_pgd
@@ -294,8 +296,9 @@ walk_puds: 
	add	end_pudp, cur_pudp, #(PTRS_PER_PUD * 8)

do_pud:	__idmap_kpti_get_pgtable_ent	pud

	tbnz	pud, #1, walk_pmds

	__idmap_kpti_put_pgtable_ent_ng	pud

next_pud:

	__idmap_kpti_put_pgtable_ent_ng	pud

skip_pud:

	add	cur_pudp, cur_pudp, 8

	cmp	cur_pudp, end_pudp

	b.ne	do_pud
@@ -314,8 +317,9 @@ walk_pmds: 
	add	end_pmdp, cur_pmdp, #(PTRS_PER_PMD * 8)

do_pmd:	__idmap_kpti_get_pgtable_ent	pmd

	tbnz	pmd, #1, walk_ptes

	__idmap_kpti_put_pgtable_ent_ng	pmd

next_pmd:

	__idmap_kpti_put_pgtable_ent_ng	pmd

skip_pmd:

	add	cur_pmdp, cur_pmdp, #8

	cmp	cur_pmdp, end_pmdp

	b.ne	do_pmd
@@ -333,7 +337,7 @@ walk_ptes: 
	add	end_ptep, cur_ptep, #(PTRS_PER_PTE * 8)

do_pte:	__idmap_kpti_get_pgtable_ent	pte

	__idmap_kpti_put_pgtable_ent_ng	pte

next_pte:

skip_pte:

	add	cur_ptep, cur_ptep, #8

	cmp	cur_ptep, end_ptep

	b.ne	do_pte

CRA Git | Maintained and supported by SUSTech CRA and CCSE