Commit 8b234274 authored by David S. Miller's avatar David S. Miller
Browse files

[SPARC64]: More TLB/TSB handling fixes. 



The SUN4V convention with non-shared TSBs is that the context
bit of the TAG is clear.  So we have to choose an "invalid"
bit and initialize new TSBs appropriately.  Otherwise a zero
TAG looks "valid".

Make sure, for the window fixup cases, that we use the right
global registers and that we don't potentially trample on
the live global registers in etrap/rtrap handling (%g2 and
%g6) and that we put the missing virtual address properly
in %g5.

Signed-off-by: default avatarDavid S. Miller <davem@davemloft.net>
parent 7adb37fe

arch/sparc64/kernel/dtlb_miss.S

+2 −2
Original line number Diff line number Diff line
@@ -2,10 +2,10 @@ 
	ldxa	[%g0] ASI_DMMU_TSB_8KB_PTR, %g1	! Get TSB 8K pointer

	ldxa	[%g0] ASI_DMMU, %g6		! Get TAG TARGET

	srlx	%g6, 48, %g5			! Get context

	sllx	%g6, 22, %g6			! Zero out context

	brz,pn	%g5, kvmap_dtlb			! Context 0 processing

	 nop					! Delay slot (fill me)

	 srlx	%g6, 22, %g6			! Delay slot

	TSB_LOAD_QUAD(%g1, %g4)			! Load TSB entry

	nop					! Push branch to next I$ line

	cmp	%g4, %g6			! Compare TAG



/* DTLB ** ICACHE line 2: TSB compare and TLB load	*/

arch/sparc64/kernel/itlb_miss.S

+6 −6
Original line number Diff line number Diff line
@@ -2,25 +2,25 @@ 
	ldxa	[%g0] ASI_IMMU_TSB_8KB_PTR, %g1	! Get TSB 8K pointer

	ldxa	[%g0] ASI_IMMU, %g6		! Get TAG TARGET

	srlx	%g6, 48, %g5			! Get context

	sllx	%g6, 22, %g6			! Zero out context

	brz,pn	%g5, kvmap_itlb			! Context 0 processing

	 nop					! Delay slot (fill me)

	 srlx	%g6, 22, %g6			! Delay slot

	TSB_LOAD_QUAD(%g1, %g4)			! Load TSB entry

	cmp	%g4, %g6			! Compare TAG

	sethi	%hi(PAGE_EXEC), %g4		! Setup exec check



/* ITLB ** ICACHE line 2: TSB compare and TLB load	*/

	sethi	%hi(PAGE_EXEC), %g4		! Setup exec check

	ldx	[%g4 + %lo(PAGE_EXEC)], %g4

	bne,pn	%xcc, tsb_miss_itlb		! Miss

	 mov	FAULT_CODE_ITLB, %g3

	andcc	%g5, %g4, %g0			! Executable?

	be,pn	%xcc, tsb_do_fault

	 nop					! Delay slot, fill me

	stxa	%g5, [%g0] ASI_ITLB_DATA_IN	! Load TLB

	retry					! Trap done

	nop



/* ITLB ** ICACHE line 3: 				*/

	nop

	nop

	stxa	%g5, [%g0] ASI_ITLB_DATA_IN	! Load TLB

	retry					! Trap done

	nop

	nop

	nop

arch/sparc64/kernel/ktlb.S

+9 −5
Original line number Diff line number Diff line
@@ -52,8 +52,10 @@ kvmap_itlb_vmalloc_addr: 


	/* Load and check PTE.  */

	ldxa		[%g5] ASI_PHYS_USE_EC, %g5

	mov		1, %g7

	sllx		%g7, TSB_TAG_INVALID_BIT, %g7

	brgez,a,pn	%g5, kvmap_itlb_longpath

	 KTSB_STORE(%g1, %g0)

	 KTSB_STORE(%g1, %g7)



	KTSB_WRITE(%g1, %g5, %g6)
@@ -146,8 +148,10 @@ kvmap_dtlb_vmalloc_addr: 


	/* Load and check PTE.  */

	ldxa		[%g5] ASI_PHYS_USE_EC, %g5

	mov		1, %g7

	sllx		%g7, TSB_TAG_INVALID_BIT, %g7

	brgez,a,pn	%g5, kvmap_dtlb_longpath

	 KTSB_STORE(%g1, %g0)

	 KTSB_STORE(%g1, %g7)



	KTSB_WRITE(%g1, %g5, %g6)
@@ -215,8 +219,8 @@ kvmap_dtlb_longpath: 
	wrpr	%g5, PSTATE_AG | PSTATE_MG, %pstate

	.section .sun4v_2insn_patch, "ax"

	.word	661b

	nop

	nop

	SET_GL(1)

	ldxa		[%g0] ASI_SCRATCHPAD, %g5

	.previous



	rdpr	%tl, %g3
@@ -226,7 +230,7 @@ kvmap_dtlb_longpath: 
	ldxa	[%g4] ASI_DMMU, %g5

	.section .sun4v_2insn_patch, "ax"

	.word	661b

	mov	%g4, %g5

	ldx	[%g5 + HV_FAULT_D_ADDR_OFFSET], %g5

	nop

	.previous

arch/sparc64/kernel/sun4v_tlb_miss.S

+21 −18
Original line number Diff line number Diff line
@@ -16,15 +16,14 @@ 
	ldx	[BASE + HV_FAULT_D_ADDR_OFFSET], VADDR; \

	ldx	[BASE + HV_FAULT_D_CTX_OFFSET], CTX;



	/* DEST = (CTX << 48) | (VADDR >> 22)

	/* DEST = (VADDR >> 22)

	 *

	 * Branch to ZERO_CTX_LABEL is context is zero.

	 */

#define	COMPUTE_TAG_TARGET(DEST, VADDR, CTX, TMP, ZERO_CTX_LABEL) \

	srlx	VADDR, 22, TMP; \

	sllx	CTX, 48, DEST; \

#define	COMPUTE_TAG_TARGET(DEST, VADDR, CTX, ZERO_CTX_LABEL) \

	srlx	VADDR, 22, DEST; \

	brz,pn	CTX, ZERO_CTX_LABEL; \

	 or	DEST, TMP, DEST;

	 nop;



	/* Create TSB pointer.  This is something like:

	 *
@@ -53,7 +52,7 @@ sun4v_itlb_miss: 
	ldxa	[%g1] ASI_SCRATCHPAD, %g1



	LOAD_ITLB_INFO(%g2, %g4, %g5)

	COMPUTE_TAG_TARGET(%g6, %g4, %g5, %g3, kvmap_itlb_4v)

	COMPUTE_TAG_TARGET(%g6, %g4, %g5, kvmap_itlb_4v)

	COMPUTE_TSB_PTR(%g1, %g4, %g3, %g7)



	/* Load TSB tag/pte into %g2/%g3 and compare the tag.  */
@@ -72,15 +71,15 @@ sun4v_itlb_miss: 
	 *

	 * %g3:	PTE

	 * %g4:	vaddr

	 * %g6:	TAG TARGET (only "CTX << 48" part matters)

	 */

sun4v_itlb_load:

	ldxa	[%g0] ASI_SCRATCHPAD, %g6

	mov	%o0, %g1		! save %o0

	mov	%o1, %g2		! save %o1

	mov	%o2, %g5		! save %o2

	mov	%o3, %g7		! save %o3

	mov	%g4, %o0		! vaddr

	srlx	%g6, 48, %o1		! ctx

	ldx	[%g6 + HV_FAULT_I_CTX_OFFSET], %o1	! ctx

	mov	%g3, %o2		! PTE

	mov	HV_MMU_IMMU, %o3	! flags

	ta	HV_MMU_MAP_ADDR_TRAP
@@ -101,7 +100,7 @@ sun4v_dtlb_miss: 
	ldxa	[%g1] ASI_SCRATCHPAD, %g1



	LOAD_DTLB_INFO(%g2, %g4, %g5)

	COMPUTE_TAG_TARGET(%g6, %g4, %g5, %g3, kvmap_dtlb_4v)

	COMPUTE_TAG_TARGET(%g6, %g4, %g5, kvmap_dtlb_4v)

	COMPUTE_TSB_PTR(%g1, %g4, %g3, %g7)



	/* Load TSB tag/pte into %g2/%g3 and compare the tag.  */
@@ -115,15 +114,15 @@ sun4v_dtlb_miss: 
	 *

	 * %g3:	PTE

	 * %g4:	vaddr

	 * %g6:	TAG TARGET (only "CTX << 48" part matters)

	 */

sun4v_dtlb_load:

	ldxa	[%g0] ASI_SCRATCHPAD, %g6

	mov	%o0, %g1		! save %o0

	mov	%o1, %g2		! save %o1

	mov	%o2, %g5		! save %o2

	mov	%o3, %g7		! save %o3

	mov	%g4, %o0		! vaddr

	srlx	%g6, 48, %o1		! ctx

	ldx	[%g6 + HV_FAULT_D_CTX_OFFSET], %o1	! ctx

	mov	%g3, %o2		! PTE

	mov	HV_MMU_DMMU, %o3	! flags

	ta	HV_MMU_MAP_ADDR_TRAP
@@ -136,10 +135,12 @@ sun4v_dtlb_load: 
	retry



sun4v_dtlb_prot:

	SET_GL(1)



	/* Load MMU Miss base into %g2.  */

	ldxa	[%g0] ASI_SCRATCHPAD, %g2

	ldxa	[%g0] ASI_SCRATCHPAD, %g5

	

	ldx	[%g2 + HV_FAULT_D_ADDR_OFFSET], %g5

	ldx	[%g5 + HV_FAULT_D_ADDR_OFFSET], %g5

	rdpr	%tl, %g1

	cmp	%g1, 1

	bgu,pn	%xcc, winfix_trampoline
@@ -189,7 +190,8 @@ sun4v_itlb_error: 
	sethi	%hi(sun4v_err_itlb_vaddr), %g1

	stx	%g4, [%g1 + %lo(sun4v_err_itlb_vaddr)]

	sethi	%hi(sun4v_err_itlb_ctx), %g1

	srlx	%g6, 48, %o1		! ctx

	ldxa	[%g0] ASI_SCRATCHPAD, %g6

	ldx	[%g6 + HV_FAULT_I_CTX_OFFSET], %o1

	stx	%o1, [%g1 + %lo(sun4v_err_itlb_ctx)]

	sethi	%hi(sun4v_err_itlb_pte), %g1

	stx	%g3, [%g1 + %lo(sun4v_err_itlb_pte)]
@@ -214,7 +216,8 @@ sun4v_dtlb_error: 
	sethi	%hi(sun4v_err_dtlb_vaddr), %g1

	stx	%g4, [%g1 + %lo(sun4v_err_dtlb_vaddr)]

	sethi	%hi(sun4v_err_dtlb_ctx), %g1

	srlx	%g6, 48, %o1		! ctx

	ldxa	[%g0] ASI_SCRATCHPAD, %g6

	ldx	[%g6 + HV_FAULT_D_CTX_OFFSET], %o1

	stx	%o1, [%g1 + %lo(sun4v_err_dtlb_ctx)]

	sethi	%hi(sun4v_err_dtlb_pte), %g1

	stx	%g3, [%g1 + %lo(sun4v_err_dtlb_pte)]

arch/sparc64/kernel/tsb.S

+11 −6
Original line number Diff line number Diff line
@@ -36,7 +36,7 @@ tsb_miss_itlb: 
	/* At this point we have:

	 * %g4 --	missing virtual address

	 * %g1 --	TSB entry address

	 * %g6 --	TAG TARGET ((vaddr >> 22) | (ctx << 48))

	 * %g6 --	TAG TARGET (vaddr >> 22)

	 */

tsb_miss_page_table_walk:

	TRAP_LOAD_PGD_PHYS(%g7, %g5)
@@ -50,8 +50,10 @@ tsb_reload: 


	/* Load and check PTE.  */

	ldxa		[%g5] ASI_PHYS_USE_EC, %g5

	mov		1, %g7

	sllx		%g7, TSB_TAG_INVALID_BIT, %g7

	brgez,a,pn	%g5, tsb_do_fault

	 TSB_STORE(%g1, %g0)

	 TSB_STORE(%g1, %g7)



	/* If it is larger than the base page size, don't

	 * bother putting it into the TSB.
@@ -62,8 +64,10 @@ tsb_reload: 
	sethi		%hi(_PAGE_SZBITS), %g7

	ldx		[%g7 + %lo(_PAGE_SZBITS)], %g7

	cmp		%g2, %g7

	mov		1, %g7

	sllx		%g7, TSB_TAG_INVALID_BIT, %g7

	bne,a,pn	%xcc, tsb_tlb_reload

	 TSB_STORE(%g1, %g0)

	 TSB_STORE(%g1, %g7)



	TSB_WRITE(%g1, %g5, %g6)
@@ -136,7 +140,7 @@ tsb_do_fault: 
	.section	.sun4v_2insn_patch, "ax"

	.word		661b

	SET_GL(1)

	ldxa		[%g0] ASI_SCRATCHPAD, %g2

	ldxa		[%g0] ASI_SCRATCHPAD, %g4

	.previous



	bne,pn		%xcc, tsb_do_itlb_fault
@@ -150,7 +154,7 @@ tsb_do_dtlb_fault: 
	ldxa	[%g4] ASI_DMMU, %g5

	.section .sun4v_2insn_patch, "ax"

	.word	661b

	ldx	[%g2 + HV_FAULT_D_ADDR_OFFSET], %g5

	ldx	[%g4 + HV_FAULT_D_ADDR_OFFSET], %g5

	nop

	.previous
@@ -217,8 +221,9 @@ tsb_flush: 
	bne,pn	%icc, 1b

	 membar	#LoadLoad

	cmp	%g1, %o1

	mov	1, %o3

	bne,pt	%xcc, 2f

	 clr	%o3

	 sllx	%o3, TSB_TAG_INVALID_BIT, %o3

	TSB_CAS_TAG(%o0, %g1, %o3)

	cmp	%g1, %o3

	bne,pn	%xcc, 1b

CRA Git | Maintained and supported by SUSTech CRA and CCSE